示例#1
0
    def gl_display(self):
        """
        use gl calls to render
        at least:
            the published position of the reference
        better:
            show the detected postion even if not published
        """

        if self.active:
            draw_points_norm([self.smooth_pos],size=15,color=RGBA(1.,1.,0.,.5))

        if self.active and self.detected:
            for e in self.candidate_ellipses:
                pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
                                    (int(e[1][0]/2),int(e[1][1]/2)),
                                    int(e[-1]),0,360,15)
                draw_polyline(pts,color=RGBA(0.,1.,0,1.))


            # lets draw an indicator on the autostop count
            e = self.candidate_ellipses[3]
            pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
                                (int(e[1][0]/2),int(e[1][1]/2)),
                                int(e[-1]),0,360,360/self.auto_stop_max)
            indicator = [e[0]] + pts[self.auto_stop:].tolist() + [e[0]]
            draw_polyline(indicator,color=RGBA(8.,0.1,0.1,.8),line_type=GL_POLYGON)
        else:
            pass
示例#2
0
    def create_mask(self, shape):
        MotionTrackedBodypart.create_mask(self, shape)

        self.wfnRoiMaskedPolarCropped = None

        if (self.mask.xMin is not None): 
            self.i_0 = self.params['gui'][self.name]['hinge']['y'] - self.roi[1]
            self.j_0 = self.params['gui'][self.name]['hinge']['x'] - self.roi[0]

    
            # Args for the ellipse calls.
            x = int(self.params['gui'][self.name]['hinge']['x'])
            y = int(self.params['gui'][self.name]['hinge']['y'])
            r_outer = int(np.ceil(self.params['gui'][self.name]['radius_outer']))
            r_inner = int(np.floor(self.params['gui'][self.name]['radius_inner']))-1
            hi = int(np.ceil(np.rad2deg(self.angle_hi_i)))
            lo = int(np.floor(np.rad2deg(self.angle_lo_i)))

    
            # Find where the mask might be clipped.  First, draw an unclipped ellipse.        
            delta = 1
            ptsUnclipped =                         cv2.ellipse2Poly((x, y), (r_outer, r_outer), 0, hi, lo, delta)
            ptsUnclipped = np.append(ptsUnclipped, cv2.ellipse2Poly((x, y), (r_inner, r_inner), 0, hi, lo, delta), 0)
            #ptsUnclipped = np.append(ptsUnclipped, [list of line1 pixels connecting the two arcs], 0) 
            #ptsUnclipped = np.append(ptsUnclipped, [list of line2 pixels connecting the two arcs], 0) 
    
    
            # These are the unclipped locations.        
            minUnclipped = ptsUnclipped.min(0)
            maxUnclipped = ptsUnclipped.max(0)
            xMinUnclipped = minUnclipped[0]
            yMinUnclipped = minUnclipped[1]
            xMaxUnclipped = maxUnclipped[0]+1
            yMaxUnclipped = maxUnclipped[1]+1

            # Compare unclipped with the as-drawn locations.
            xClip0 = self.mask.xMin - xMinUnclipped
            yClip0 = self.mask.yMin - yMinUnclipped
            xClip1 = xMaxUnclipped - self.mask.xMax
            yClip1 = yMaxUnclipped - self.mask.yMax
            roiClipped = np.array([xClip0, yClip0, xClip1, yClip1])
    
            (i_n, j_n) = shape[:2]
                    
            # Determine how much of the bottom of the polar image to trim off (i.e. rClip) based on if the ellipse is partially offimage.
            (rClip0, rClip1, rClip2, rClip3) = (1.0, 1.0, 1.0, 1.0)
            if (roiClipped[0]>0): # Left
                rClip0 = 1.0 - (float(roiClipped[0])/float(r_outer-r_inner))#self.j_0))
            if (roiClipped[1]>0): # Top
                rClip1 = 1.0 - (float(roiClipped[1])/float(r_outer-r_inner))#self.i_0))
            if (roiClipped[2]>0): # Right
                rClip2 = 1.0 - (float(roiClipped[2])/float(r_outer-r_inner))#j_n-self.j_0))
            if (roiClipped[3]>0): # Bottom
                rClip3 = 1.0 - (float(roiClipped[3])/float(r_outer-r_inner))#i_n-self.i_0))
    
            self.rClip = np.min([rClip0, rClip1, rClip2, rClip3])
 def drawOn(self, canvas, color=(0, 0, 255), degreeDelta=20):
     points = cv2.ellipse2Poly(tuple(map(int, self.position)),
                               tuple(map(int, self.axes)),
                               self.angle*180/math.pi,
                               0, 360,
                               degreeDelta)
     cv2.fillConvexPoly(canvas, points, color, cv2.CV_AA)
    def gl_display(self):
        """
        use gl calls to render
        at least:
            the published position of the reference
        better:
            show the detected postion even if not published
        """

        # debug mode within world will show green ellipses around detected ellipses
        if self.active:
            for marker in self.markers:
                e = marker["ellipses"][-1]  # outermost ellipse
                pts = cv2.ellipse2Poly(
                    (int(e[0][0]), int(e[0][1])),
                    (int(e[1][0] / 2), int(e[1][1] / 2)),
                    int(e[-1]),
                    0,
                    360,
                    15,
                )
                draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0))
                if len(self.markers) > 1:
                    draw_polyline(
                        pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON
                    )
def draw_hand(full_img, joint_coords, is_loss_track):
    if is_loss_track:
        joint_coords = FLAGS.default_hand

    # Plot joints
    for joint_num in range(FLAGS.num_of_joints):
        color_code_num = (joint_num // 4)
        if joint_num in [0, 4, 8, 12, 16]:
            joint_color = list(map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
            cv2.circle(full_img, center=(int(joint_coords[joint_num][1]), int(joint_coords[joint_num][0])), radius=3,
                       color=joint_color, thickness=-1)
        else:
            joint_color = list(map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
            cv2.circle(full_img, center=(int(joint_coords[joint_num][1]), int(joint_coords[joint_num][0])), radius=3,
                       color=joint_color, thickness=-1)

    # Plot limbs
    for limb_num in range(len(FLAGS.limbs)):
        x1 = int(joint_coords[int(FLAGS.limbs[limb_num][0])][0])
        y1 = int(joint_coords[int(FLAGS.limbs[limb_num][0])][1])
        x2 = int(joint_coords[int(FLAGS.limbs[limb_num][1])][0])
        y2 = int(joint_coords[int(FLAGS.limbs[limb_num][1])][1])
        length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
        if length < 150 and length > 5:
            deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
            polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
                                       (int(length / 2), 3),
                                       int(deg),
                                       0, 360, 1)
            color_code_num = limb_num // 4
            limb_color = list(map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num]))
            cv2.fillConvexPoly(full_img, polygon, color=limb_color)
    def _visualize(self, img, joint_cands_indices, all_peaks, candidate):

        cmap = matplotlib.cm.get_cmap('hsv')
        for i in range(len(self.index2limbname)-1):
            rgba = np.array(cmap(1 - i / 18. - 1. / 36))
            rgba[0:3] *= 255
            for j in range(len(all_peaks[i])):
                cv2.circle(img, (int(all_peaks[i][j][0]), int(
                    all_peaks[i][j][1])), 4, self.colors[i], thickness=-1)

        stickwidth = 4
        for i in range(len(self.index2limbname) - 2):
            for joint_cand_indices in joint_cands_indices:
                index = joint_cand_indices[np.array(self.limb_sequence[i],
                                                    dtype=np.int32) - 1]
                if -1 in index:
                    continue
                cur_img = img.copy()
                Y = candidate[index.astype(int), 0]
                X = candidate[index.astype(int), 1]
                mX = np.mean(X)
                mY = np.mean(Y)
                length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
                angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
                polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(
                    length / 2), stickwidth), int(angle), 0, 360, 1)
                cv2.fillConvexPoly(cur_img, polygon, self.colors[i])
                img = cv2.addWeighted(img, 0.4, cur_img, 0.6, 0)

        return img
示例#7
0
    def gl_display(self):
        """
        use gl calls to render
        at least:
            the published position of the reference
        better:
            show the detected postion even if not published
        """

        if self.active and self.detected:
            for e in self.candidate_ellipses:
                pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
                                    (int(e[1][0]/2),int(e[1][1]/2)),
                                    int(e[-1]),0,360,15)
                draw_gl_polyline(pts,(0.,1.,0,1.))
        else:
            pass
        if self._window:
            self.gl_display_in_window()


        if not self.active and self.error_lines is not None:
            draw_gl_polyline_norm(self.error_lines,(1.,0.5,0.,.5),type='Lines')
            draw_gl_points_norm(self.error_lines[1::2],color=(.0,0.5,0.5,.5),size=3)
            draw_gl_points_norm(self.error_lines[0::2],color=(.5,0.0,0.0,.5),size=3)
    def gl_display(self):
        """
        use gl calls to render
        at least:
            the published position of the reference
        better:
            show the detected postion even if not published
        """

        if self.active:
            # draw the largest ellipse of all detected markers
            for marker in self.markers:
                e = marker["ellipses"][-1]
                pts = cv2.ellipse2Poly(
                    (int(e[0][0]), int(e[0][1])),
                    (int(e[1][0] / 2), int(e[1][1] / 2)),
                    int(e[-1]),
                    0,
                    360,
                    15,
                )
                draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0))
                if len(self.markers) > 1:
                    draw_polyline(
                        pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON
                    )

            # draw indicator on the stop marker(s)
            if self.auto_stop:
                for marker in self.markers:
                    if marker["marker_type"] == "Stop":
                        e = marker["ellipses"][-1]
                        pts = cv2.ellipse2Poly(
                            (int(e[0][0]), int(e[0][1])),
                            (int(e[1][0] / 2), int(e[1][1] / 2)),
                            int(e[-1]),
                            0,
                            360,
                            360 // self.auto_stop_max,
                        )
                        indicator = [e[0]] + pts[self.auto_stop :].tolist() + [e[0]]
                        draw_polyline(
                            indicator,
                            color=RGBA(8.0, 0.1, 0.1, 0.8),
                            line_type=gl.GL_POLYGON,
                        )
示例#9
0
	def draw_all_ellipses(self):
		# draws all ellipses in self.ellipses.
		glPushMatrix()
		for ellipse in self.ellipses[-10:]:
			glColor3f(0.0, 1.0, 0.0)  #set color to green
			pts = cv2.ellipse2Poly( (int(ellipse.center[0]),int(ellipse.center[1])),
                                        (int(ellipse.major_radius),int(ellipse.minor_radius)),
                                        int(ellipse.angle*180/scipy.pi),0,360,15)
			draw_polyline(pts,2,color = RGBA(0,1,1,.5))
		glPopMatrix()
示例#10
0
def polytest(x,y,rx,ry,rw,rh,rang):
    points=cv2.ellipse2Poly(
            (rx,ry),
            axes=(rw/2,rh/2),
            angle=rang,
            arcStart=0,
            arcEnd=360,
            delta=3
        )
    return cv2.pointPolygonTest(np.array(points), (x,y), measureDist=1)
示例#11
0
 def draw_circle(self, frame, pos,r,c):
     ''' function that is called by draw_marker exclusively ''' 
     pts = cv2.ellipse2Poly(tuple(pos),(r,r),0,0,360,10)
     draw_gl_polyline(pts,c,'Polygon')
     world_size =  self.world_size
     window_size = glfwGetWindowSize(self._window)
     ratio = [float(world_size[0])/window_size[0], float(world_size[1])/window_size[1]]
     pos_0 = ratio[0] * pos[0] * .75 + world_size[0] * (1 - .75)/2
     pos_1 = ratio[1] * pos[1] * .75 + world_size[1] * (1 - .75)/2
     pos_1 = int(pos_0), int(pos_1)
     c = cv2.cv.CV_RGB(c[0]*255, c[1]*255, c[2]*255)
     cv2.ellipse(frame, tuple(pos_1),(r,r),0,0,360, c)
示例#12
0
文件: gazes.py 项目: hanke/vilay
 def get_each(self, stimulus, history_length = 0, approx = False):
     ovImg = np.zeros((stimulus.height, stimulus.width,3),'uint8')
     colors = utils.autumn(len(self.gazes))
     
     gaze_coords = self.getGazeMovieCoords(stimulus, history_length, approx)
     gaze_coords = sorted(gaze_coords,key=operator.itemgetter(3),reverse=True)
     for (i, x, y, prio) in gaze_coords:
         if prio == 1:
             cv2.circle(ovImg, (int(x*stimulus.width),int(y*stimulus.height)), 15, colors[int(i)])
         pts = cv2.ellipse2Poly((int(x*stimulus.width),int(y*stimulus.height)),(2,2),0,0,360,int(360/6))
         opacity = float(history_length-prio+2) / (history_length+1)
         cv2.fillConvexPoly(ovImg, pts, (int(colors[int(i)][0]*opacity), int(colors[int(i)][1]*opacity), int(colors[int(i)][2]*opacity)))
     return ovImg    
示例#13
0
def ellipse2Rect(params):
    rad_x = params[0]
    rad_y = params[1]
    angle = params[2] * 180.0 / pi
    center_x = params[3]
    center_y = params[4]
    pts = cv.ellipse2Poly((int(center_x), int(center_y)), (int(rad_x), int(rad_y)),
                          int(angle), 0, 360, 10)
    rect = cv.boundingRect(pts)
    left = rect[0]
    top = rect[1]
    right = rect[0] + rect[2]
    bottom = rect[1] + rect[3]
    return left, top, right, bottom
def draw_limbs_2d(img, joints_2d, limb_parents):
    for limb_num in range(len(limb_parents)-1):
        x1 = joints_2d[limb_num, 0]
        y1 = joints_2d[limb_num, 1]
        x2 = joints_2d[limb_parents[limb_num], 0]
        y2 = joints_2d[limb_parents[limb_num], 1]
        length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
        # if length < 10000 and length > 5:
        deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
        polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
                                   (int(length / 2), 3),
                                   int(deg),
                                   0, 360, 1)
        cv2.fillConvexPoly(img, polygon, color=(0,255,0))
    return img
def parse_json(json_file, image_file):
    canvas = cv2.imread(image_file) # B,G,R order

    with open(json_file) as infile:
        info = json.load(infile)

    pose_list = list()
    count = 1
    for item in info['persons']:
        count += 1
        pose_list.append(item['keypoints'])

    pose_array = np.array(pose_list)
    print pose_array.shape

    cmap = matplotlib.cm.get_cmap('hsv')
    
    for i in range(18):
        rgba = np.array(cmap(1 - i/18. - 1./36))
        rgba[0:3] *= 255
        for j in range(pose_array.shape[0]):
            if pose_array[j][i][2] == 2:
                continue

            cv2.circle(canvas, (int(pose_array[j][i][0]), int(pose_array[j][i][1])), 4, colors[i], thickness=-1)

    cv2.imshow("main", canvas)
    cv2.waitKey()

    for i in range(17):
        for n in range(pose_array.shape[0]):
            Y = pose_array[n][np.array(limbSeq[i])-1][:,0]
            X = pose_array[n][np.array(limbSeq[i])-1][:,1]
            if 2. in pose_array[n][np.array(limbSeq[i])-1][:,2]:
                continue

            cur_canvas = canvas.copy()

            mX = np.mean(X)
            mY = np.mean(Y)
            length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
            angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
            polygon = cv2.ellipse2Poly((int(mY),int(mX)), (int(length/2), 4), int(angle), 0, 360, 1)
            cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
            canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)

    cv2.imshow("main", canvas)
    cv2.waitKey()
    def gl_display(self):
        """
        use gl calls to render
        at least:
            the published position of the reference
        better:
            show the detected postion even if not published
        """

        if self.active and self.detected:
            for e in self.candidate_ellipses:
                pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
                                    (int(e[1][0]/2),int(e[1][1]/2)),
                                    int(e[-1]),0,360,15)
                draw_gl_polyline(pts,(0.,1.,0,1.))
        else:
            pass
def link_key_point(img_canvas, candidate, subset, stickwidth=4):
    for i in range(17):
        for n in range(len(subset)):
            index = subset[n][np.array(limb_seq[i]) - 1]
            if -1 in index:
                continue
            cur_canvas = img_canvas.copy()
            Y = candidate[index.astype(int), 0]
            X = candidate[index.astype(int), 1]
            mX = np.mean(X)
            mY = np.mean(Y)
            length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
            angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
            polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
            cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
            img_canvas = cv2.addWeighted(img_canvas, 0.4, cur_canvas, 0.6, 0)

    return img_canvas
示例#18
0
 def getPoly(self, typ, pts):
     if typ == 0:
         return []
     elif typ == 1:
         x = pts[:,0]
         y = pts[:,1]
         
         x.sort()
         y.sort()
         return np.array([[x[0],y[0]],[x[0],y[1]],[x[1],y[1]],[x[1],y[0]]])
     elif typ == 2:
         r = int(np.linalg.norm(pts[0,:] - pts[1,:]))
         return cv2.ellipse2Poly((pts[0,0],pts[0,1]), (r,r), 0, 0, 360, 10)
     elif typ == 3:
         return pts[0:-2,:]
     elif typ == 4:
         print "Warning: Place->getPoly(): typ=4 (ellipse) not implemented yet"
         return None
示例#19
0
    def gl_display(self):
        """
        use gl calls to render
        at least:
            the published position of the reference
        better:
            show the detected postion even if not published
        """

        if self.active and self.detected:
            draw_points_norm([self.smooth_pos],size=15,color=RGBA(1.,1.,0.,.5))
            for e in self.candidate_ellipses:
                pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
                                       (int(e[1][0]/2),int(e[1][1]/2)),
                                       int(e[-1]),0,360,15)
                draw_polyline(pts,color=RGBA(0.,1.,0,1.))

        else:
            pass
    def visualize(self,all_peaks, image_path, subset, candidate, dest_points, dest_skeleton):
        limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], [10, 11], [2, 12], [12, 13],
                   [13, 14], [2, 1], [1, 15], [15, 17], [1, 16], [16, 18], [3, 17], [6, 18]]
        colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0],
                  [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255],
                  [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
        cmap = matplotlib.cm.get_cmap('hsv')

        oriImg = cv.imread(image_path)
        canvas = cv.imread(image_path)

        for i in range(18):
            rgba = np.array(cmap(1 - i / 18. - 1. / 36))
            rgba[0:3] *= 255
            for j in range(len(all_peaks[i])):
                cv.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)

        to_plot = cv.addWeighted(oriImg, 0.3, canvas, 0.7, 0)
        #cv.imwrite(dest_points, to_plot)
        #plt.imshow(to_plot[:, :, [2, 1, 0]])
        #plt.show()


        stickwidth = 4

        for i in range(17):
            for n in range(len(subset)):
                index = subset[n][np.array(limbSeq[i]) - 1]
                if -1 in index:
                    continue
                cur_canvas = canvas.copy()
                Y = candidate[index.astype(int), 0]
                X = candidate[index.astype(int), 1]
                mX = np.mean(X)
                mY = np.mean(Y)
                length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
                angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
                polygon = cv.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
                cv.fillConvexPoly(cur_canvas, polygon, colors[i])
                canvas = cv.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)

        cv.imwrite(dest_skeleton, canvas)
    def __link_key_point(self, img_canvas, candidate, subset):
        for i in range(self.configer.get('data', 'num_keypoints')-1):
            for n in range(len(subset)):
                index = subset[n][np.array(self.configer.get('details', 'limb_seq')[i]) - 1]
                if -1 in index:
                    continue
                cur_canvas = img_canvas.copy()
                Y = candidate[index.astype(int), 0]
                X = candidate[index.astype(int), 1]
                mX = np.mean(X)
                mY = np.mean(Y)
                length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
                angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
                polygon = cv2.ellipse2Poly((int(mY), int(mX)),
                                           (int(length / 2),
                                            self.configer.get('vis', 'stick_width')), int(angle), 0, 360, 1)
                cv2.fillConvexPoly(cur_canvas, polygon, self.configer.get('details', 'color_list')[i])
                img_canvas = cv2.addWeighted(img_canvas, 0.4, cur_canvas, 0.6, 0)

        return img_canvas
示例#22
0
    def gl_display(self):
        """
        use gl calls to render
        at least:
            the published position of the reference
        better:
            show the detected postion even if not published
        """

        # debug mode within world will show green ellipses around detected ellipses
        if self.active and self.detected:
            for e in self.candidate_ellipses:
                pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
                                    (int(e[1][0]/2),int(e[1][1]/2)),
                                    int(e[-1]),0,360,15)
                draw_polyline(pts,1,RGBA(0.,1.,0.,1.))

        else:
            pass
        if self._window:
            self.gl_display_in_window()
示例#23
0
def get_withinring(ringpolyTVArray, timepoints, x, y):
    
    rx = ringpolyTVArray[:,0].astype(np.int)
    ry = ringpolyTVArray[:,1].astype(np.int)
    rw = ringpolyTVArray[:,2].astype(np.int)
    rh = ringpolyTVArray[:,3].astype(np.int)
    rang = ringpolyTVArray[:,4].astype(np.int)
    # poly test
    peaks_within = []
    for p in timepoints:
        points=cv2.ellipse2Poly(
                (rx[p],ry[p]),
                axes=(rw[p]/2,rh[p]/2),
                angle=rang[p],
                arcStart=0,
                arcEnd=360,
                delta=3
            )
        inout = cv2.pointPolygonTest(np.array(points), (x[p],y[p]), measureDist=1)
        if inout > 0:
            peaks_within.append(p)

    return peaks_within
def draw_circle(pos,r,c):
    pts = cv2.ellipse2Poly(tuple(pos),(r,r),0,0,360,10)
    draw_gl_polyline(pts,c,'Polygon')
示例#25
0
def renderface(facepts, canvas, disp=False, threshold = 0.2, smalldot = 2):
	if disp:
		color = tuple([255, 255, 255])
	else:
		color = tuple([0, 0, 0])

	avecons = sum(facepts[2:len(facepts):3]) / 70.0

	if avecons < threshold:
		return canvas
	i = 0

	while i < 210:
		confidence = facepts[i+2]
		if confidence > 0:
			cv.circle(canvas, (int(facepts[i]), int(facepts[i+1])), smalldot, color, thickness=-1)
		i += 3

	if disp: #graph the lines between points
		stickwidth = 1
		linearSeq = [range(0, 16+1), range(17, 21+1), range(22, 26+1), range(27, 30+1), range(31, 35+1)]
		circularSeq = [range(36, 41+1), range(42, 47+1), range(48, 59+1), range(60, 67)]

		for line in linearSeq:
			for step in line:
				if step != line[len(line) - 1]:
					firstlimb_ind = step
					secondlimb_ind = step + 1
					if (facepts[3*firstlimb_ind + 2] > 0) and (facepts[3*secondlimb_ind + 2] > 0):
						cur_canvas = canvas.copy()
						Y = [facepts[3*firstlimb_ind], facepts[3*secondlimb_ind]]
						X = [facepts[3*firstlimb_ind + 1], facepts[3*secondlimb_ind + 1]]
						mX = np.mean(X)
						mY = np.mean(Y)
						length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
						angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
						polygon = cv.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
						cv.fillConvexPoly(cur_canvas, polygon, color)
						canvas = cv.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)

		for circle in circularSeq:
			for step in circle:
				if step == circle[len(circle) - 1]:
					firstlimb_ind = step
					secondlimb_ind = circle[0]
				else:
					firstlimb_ind = step
					secondlimb_ind = step + 1
				
				if (facepts[3*firstlimb_ind + 2] > 0) and (facepts[3*secondlimb_ind + 2] > 0):
					cur_canvas = canvas.copy()
					Y = [facepts[3*firstlimb_ind], facepts[3*secondlimb_ind]]
					X = [facepts[3*firstlimb_ind + 1], facepts[3*secondlimb_ind + 1]]
					mX = np.mean(X)
					mY = np.mean(Y)
					length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
					angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
					polygon = cv.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
					cv.fillConvexPoly(cur_canvas, polygon, color)
					canvas = cv.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)

	return canvas
def do_work(test_image,save_name):
    oriImg = cv2.imread(test_image) # B,G,R order
    multiplier = [x * model['boxsize'] / oriImg.shape[0] for x in param['scale_search']]
    heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
    paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))

    print 'len of multipler'
    print len(multiplier)
    for m in range(len(multiplier)):
        scale = multiplier[m]
        print scale
        imageToTest = cv2.resize(oriImg, (0,0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
        imageToTest_padded, pad = util.padRightDownCorner(imageToTest, model['stride'], model['padValue'])
        print imageToTest_padded.shape
        
        net.blobs['data'].reshape(*(1, 3, imageToTest_padded.shape[0], imageToTest_padded.shape[1]))
        #net.forward() # dry run
        net.blobs['data'].data[...] = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,2,0,1))/256 - 0.5;
        start_time = time.time()
        output_blobs = net.forward()
        print('At scale %d, The CNN took %.2f ms.' % (m, 1000 * (time.time() - start_time)))
        start_time = time.time()
        # extract outputs, resize, and remove padding
        heatmap = np.transpose(np.squeeze(net.blobs[output_blobs.keys()[1]].data), (1,2,0)) # output 1 is heatmaps
        heatmap = cv2.resize(heatmap, (0,0), fx=model['stride'], fy=model['stride'], interpolation=cv2.INTER_CUBIC)
        heatmap = heatmap[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
        heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
        
        paf = np.transpose(np.squeeze(net.blobs[output_blobs.keys()[0]].data), (1,2,0)) # output 0 is PAFs
        paf = cv2.resize(paf, (0,0), fx=model['stride'], fy=model['stride'], interpolation=cv2.INTER_CUBIC)
        paf = paf[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
        paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)    
        
        heatmap_avg = heatmap_avg + heatmap / len(multiplier)
        paf_avg = paf_avg + paf / len(multiplier)
        print('2 At scale %d, The CNN took %.2f ms.' % (m, 1000 * (time.time() - start_time)))

    start_time = time.time()
    all_peaks = []
    peak_counter = 0

    for part in range(19-1):
        x_list = []
        y_list = []
        map_ori = heatmap_avg[:,:,part]
        map = gaussian_filter(map_ori, sigma=3)
        
        map_left = np.zeros(map.shape)
        map_left[1:,:] = map[:-1,:]
        map_right = np.zeros(map.shape)
        map_right[:-1,:] = map[1:,:]
        map_up = np.zeros(map.shape)
        map_up[:,1:] = map[:,:-1]
        map_down = np.zeros(map.shape)
        map_down[:,:-1] = map[:,1:]
        
        peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param['thre1']))
        peaks = zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]) # note reverse
        peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks]
        id = range(peak_counter, peak_counter + len(peaks))
        peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))]

        all_peaks.append(peaks_with_score_and_id)
        peak_counter += len(peaks)
        

              
    connection_all = []
    special_k = []
    mid_num = 10

    for k in range(len(mapIdx)):
        score_mid = paf_avg[:,:,[x-19 for x in mapIdx[k]]]
        candA = all_peaks[limbSeq[k][0]-1]
        candB = all_peaks[limbSeq[k][1]-1]
        nA = len(candA)
        nB = len(candB)
        indexA, indexB = limbSeq[k]
        if(nA != 0 and nB != 0):
            connection_candidate = []
            for i in range(nA):
                for j in range(nB):
                    vec = np.subtract(candB[j][:2], candA[i][:2])
                    norm = math.sqrt(vec[0]*vec[0] + vec[1]*vec[1])
                    vec = np.divide(vec, norm)
                    
                    startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
                                   np.linspace(candA[i][1], candB[j][1], num=mid_num))
                    
                    vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
                                      for I in range(len(startend))])
                    vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
                                      for I in range(len(startend))])

                    score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
                    score_with_dist_prior = sum(score_midpts)/len(score_midpts) + min(0.5*oriImg.shape[0]/norm-1, 0)
                    criterion1 = len(np.nonzero(score_midpts > param['thre2'])[0]) > 0.8 * len(score_midpts)
                    criterion2 = score_with_dist_prior > 0
                    if criterion1 and criterion2:
                        connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior+candA[i][2]+candB[j][2]])

            connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
            connection = np.zeros((0,5))
            for c in range(len(connection_candidate)):
                i,j,s = connection_candidate[c][0:3]
                if(i not in connection[:,3] and j not in connection[:,4]):
                    connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
                    if(len(connection) >= min(nA, nB)):
                        break

            connection_all.append(connection)
        else:
            special_k.append(k)
            connection_all.append([])
            
    # last number in each row is the total parts number of that person
    # the second last number in each row is the score of the overall configuration
    subset = -1 * np.ones((0, 20))
    candidate = np.array([item for sublist in all_peaks for item in sublist])

    for k in range(len(mapIdx)):
        if k not in special_k:
            partAs = connection_all[k][:,0]
            partBs = connection_all[k][:,1]
            indexA, indexB = np.array(limbSeq[k]) - 1

            for i in range(len(connection_all[k])): #= 1:size(temp,1)
                found = 0
                subset_idx = [-1, -1]
                for j in range(len(subset)): #1:size(subset,1):
                    if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
                        subset_idx[found] = j
                        found += 1
                
                if found == 1:
                    j = subset_idx[0]
                    if(subset[j][indexB] != partBs[i]):
                        subset[j][indexB] = partBs[i]
                        subset[j][-1] += 1
                        subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
                elif found == 2: # if found 2 and disjoint, merge them
                    j1, j2 = subset_idx
                    print "found = 2"
                    membership = ((subset[j1]>=0).astype(int) + (subset[j2]>=0).astype(int))[:-2]
                    if len(np.nonzero(membership == 2)[0]) == 0: #merge
                        subset[j1][:-2] += (subset[j2][:-2] + 1)
                        subset[j1][-2:] += subset[j2][-2:]
                        subset[j1][-2] += connection_all[k][i][2]
                        subset = np.delete(subset, j2, 0)
                    else: # as like found == 1
                        subset[j1][indexB] = partBs[i]
                        subset[j1][-1] += 1
                        subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]

                # if find no partA in the subset, create a new subset
                elif not found and k < 17:
                    row = -1 * np.ones(20)
                    row[indexA] = partAs[i]
                    row[indexB] = partBs[i]
                    row[-1] = 2
                    row[-2] = sum(candidate[connection_all[k][i,:2].astype(int), 2]) + connection_all[k][i][2]
                    subset = np.vstack([subset, row])

    # delete some rows of subset which has few parts occur
    deleteIdx = [];
    for i in range(len(subset)):
        if subset[i][-1] < 4 or subset[i][-2]/subset[i][-1] < 0.4:
            deleteIdx.append(i)
    subset = np.delete(subset, deleteIdx, axis=0)


    cmap = matplotlib.cm.get_cmap('hsv')

    canvas = cv2.imread(test_image) # B,G,R order

    for i in range(18):
        rgba = np.array(cmap(1 - i/18. - 1./36))
        rgba[0:3] *= 255
        for j in range(len(all_peaks[i])):
            cv2.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
    # visualize 2
    stickwidth = 4

    for i in range(17):
        for n in range(len(subset)):
            index = subset[n][np.array(limbSeq[i])-1]
            if -1 in index:
                continue
            cur_canvas = canvas.copy()
            Y = candidate[index.astype(int), 0]
            X = candidate[index.astype(int), 1]
            mX = np.mean(X)
            mY = np.mean(Y)
            length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
            angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
            polygon = cv2.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
            cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
            canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
    cv2.imwrite(save_name,canvas)        
    print('3 At scale %d, The CNN took %.2f ms.' % (m, 1000 * (time.time() - start_time)))
示例#27
0
	def extend_joint_area(self, joint, radius=65):
		x = int(round(joint[0])
		y = int(round(joint[1])
		
		left = x - radius
		top = y - radius
		right = x + radius
		bottom = y + radius
		#cropped_img = image[y-radius:y+radius, x-radius:x+radius]
		return list(zip(left, top, right, bottom))
	
	def calculate_HOI(self):
		self.relations = []
		
		joints = self.pose.get_pose()
		
		if not check_for_person(joints):
			print("No person found")
		
		object_data = self.object.predict() #Image should be saved elsewhere
		
		#Each check returns the object list any objects that were found removed from the next check (stops items being held and eaten at the same time)
		object_data = check_for_face_interactions(object_data, joints)
		object_data = check_for_hand_interactions(object_data, joints)
		object_data = check_for_misc_interactions(object_data, joints)
		
		print("Unused objects: " + object_data)
		
		return self.relations

class Pose_Mirroring():

	def __init__(self, pose, model_dir="D:/RobotMaster/Models", model_name="InceptionV3.hdf5"):
		self.pose = pose
		self.model = init_custom_pose_model(model_dir, model_name)
		self.part_pairs = [1,2,   1,5,   2,3,   3,4,   5,6,   6,7,   1,0,   0,15,   15,17]
		self.colours = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
          [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
          [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
		pass
	
	#Done - untested
	def init_custom_pose_model(self, dir, name):
		try:
			model = load_model(os.path.join(dir, name))
			return model
		except:
			raise Exception("Model not found")

	def get_pose(self, frame):
		keypoints = pose.get_pose(frame)
		num_people = len(keypoints)
		if num_people == 0:
			print("No people found")
			return None
		
		#Only run on the first skeleton found, not multiple people
		frame = self.draw_person(frame, keypoints[0])
		label = self.predict(frame)
		label = self.label_to_name("D:/RobotMaster/res/legend.txt", label
		
		return label
	
	def label_to_name(self, legend, label_val):
		with open(legend, 'r') as f:
			lines = f.readlines()
			return lines[int(label_num].split(",")[1]
	
	def predict(self, img, img_target_size=299):
		img = cv2.resize(img, (img_target_size, img_target_size))
		x = np.expand_dims(img, axis=0)
		x = preprocess_input(x.astype(float))
		pred = self.model.predict(x)
		pred = pred.tolist()
		pred = pred[0]
		return pred.index(max(pred))
	
	def draw_person(self, img, person):
		partial_joints = [0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 17, 18]
		upper_skeleton_img = np.zeros(img.shape[0], img.shape[1], 3), np.unit8)
		counter = 0
		
		for data in person:
			x, y, score = data
			if x > 0 and y > 0:
				if counter in partial_joints:
					cv2.cricle(upper_skeleton_img, (round(x), round(y)), 7, (0, 0, 255), -1)
					
			counter += 1
		
		upper_skeleton_img = visualise_person(upper_skeleton_img, person)
		cropped_img = crop_person(upper_skeleton_img, person, partial_joints)
		return cropped_img
	
	def crop_person(self, img, person, upper_indexs):
		minx = 99999
		miny = 99999
		maxx = 0
		maxy = 0
		counter = 0
		
		for joint in person:
			x, y, score = joint
			if counter in partial_joints:
				if x > 0
					if round(x) > maxx:
						maxx = x + 10
					if round(x) < minx:
						minx = x - 10
					if round(y) > maxy:
						maxy = y + 10
					if round(y) < miny:
						miny = y - 10
			counter += 1
		cropped = img[int(round(miny)):int(round(maxy)), int(round(minx)):int(round(maxx))]
		return cropped
	
	def visualise_person(self, img, person):
		pairs = self.part_pairs
		stickwidth = 4
		cur_img = img.copy()
		counter = 0;
		for i in range(0, len(pairs),2):   
			if person[pairs[i],0] > 0 and person[pairs[i+1],0] > 0:
				Y = [person[pairs[i],0], person[pairs[i+1],0]]
				X = [person[pairs[i],1], person[pairs[i+1],1]]
				mX = np.mean(X)
				mY = np.mean(Y)
				length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
				angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
				polygon = cv2.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
				cv2.fillConvexPoly(cur_img, polygon, colours[counter])
				counter = counter + 1
		
		img = cv2.addWeighted(img, 0.4, cur_img, 0.6, 0)
		return img
	
class Age_Gender():
	def __init__(self, dir="D:/NAORobot/RossProject/"):
		self.bat_file_dir = dir
		pass
	
	def start(self):
		Popen(os.path.join(self.dir, "CompleteBuild.bat"))

class Secret_Project():
	def __init__(self):
		self.lower_blue = np.array([0, 102, 255])
		self.upper_blue = np.array([0, 0, 204])
		pass
	
	def apply_mask(self, img):
		hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
		mask = cv2.inRange(hsv, self.lower_blue, self.upper_blue)
		res = cv2.bitwise_and(img,img, mask=mask)
		
		kernel = np.ones((15,15),np.float32)/225
		smoothed = cv2.filter2D(res,-1,kernel)
		
		_, secret = cv2.threshold(smoothed, 30, 255, cv2.THRESH_BINARY)
		
		cv2.imshow("average", smoothed)
		cv2.imshow("secret", secret)
		
		if cv2.waitKey(1) & 0xFF == 27:
			cv2.destroyAllWindows()
			return False
	
	def reposition(self):
		
		pass
def main(argv):
    global joint_detections
    os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)

    """ Initial tracker
    """
    tracker = tracking_module.SelfTracker([FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)

    """ Build network graph
    """
    model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
                                heatmap_size=FLAGS.heatmap_size,
                                stages=FLAGS.cpm_stages,
                                joints=FLAGS.num_of_joints,
                                img_type=FLAGS.color_channel,
                                is_training=False)
    saver = tf.train.Saver()

    """ Get output node
    """
    output_node = tf.get_default_graph().get_tensor_by_name(name=FLAGS.output_node_names)

    device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
    sess_config = tf.ConfigProto(device_count=device_count)
    sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
    sess_config.gpu_options.allow_growth = True
    sess_config.allow_soft_placement = True
    with tf.Session(config=sess_config) as sess:

        model_path_suffix = os.path.join(FLAGS.network_def,
                                         'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
                                         'joints_{}'.format(FLAGS.num_of_joints),
                                         'stages_{}'.format(FLAGS.cpm_stages),
                                         'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr, FLAGS.lr_decay_rate,
                                                                          FLAGS.lr_decay_step)
                                         )
        model_save_dir = os.path.join('models',
                                      'weights',
                                      model_path_suffix)
        print('Load model from [{}]'.format(os.path.join(model_save_dir, FLAGS.model_path)))
        if FLAGS.model_path.endswith('pkl'):
            model.load_weights_from_file(FLAGS.model_path, sess, False)
        else:
            saver.restore(sess, 'models/weights/cpm_hand')

        # Check weights
        for variable in tf.global_variables():
            with tf.variable_scope('', reuse=True):
                var = tf.get_variable(variable.name.split(':0')[0])
                print(variable.name, np.mean(sess.run(var)))

        # Create webcam instance
        if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
            cam = cv2.VideoCapture(FLAGS.cam_id)

        # Create kalman filters
        if FLAGS.use_kalman:
            kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)]
            for _, joint_kalman_filter in enumerate(kalman_filter_array):
                joint_kalman_filter.transitionMatrix = np.array(
                    [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
                    np.float32)
                joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
                joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
                                                               np.float32) * FLAGS.kalman_noise
        else:
            kalman_filter_array = None

        if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
            test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
            test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))

            test_img_input = normalize_and_centralize_img(test_img_resize)

            t1 = time.time()
            predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
                                                          output_node,
                                                          ],
                                                         feed_dict={model.input_images: test_img_input}
                                                         )
            print('fps: %.2f' % (1 / (time.time() - t1)))

            correct_and_draw_hand(test_img,
                                  cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)),
                                  kalman_filter_array, tracker, tracker.input_crop_ratio, test_img)

            # Show visualized image
            # demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array)
            cv2.imshow('demo_img', test_img.astype(np.uint8))
            cv2.waitKey(0)

        elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']:
            while True:
                # # Prepare input image
                _, full_img = cam.read()

                test_img = tracker.tracking_by_joints(full_img, joint_detections=joint_detections)
                crop_full_scale = tracker.input_crop_ratio
                test_img_copy = test_img.copy()

                # White balance
                test_img_wb = utils.img_white_balance(test_img, 5)
                test_img_input = normalize_and_centralize_img(test_img_wb)

                # Inference
                t1 = time.time()
                stage_heatmap_np = sess.run([output_node],
                                            feed_dict={model.input_images: test_img_input})
                print('FPS: %.2f' % (1 / (time.time() - t1)))

                local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale,
                                             test_img_copy)

                cv2.imshow('local_img', local_img.astype(np.uint8))
                cv2.imshow('global_img', full_img.astype(np.uint8))
                if cv2.waitKey(1) == ord('q'): break


        elif FLAGS.DEMO_TYPE == 'Joint_HM':
            while True:
                # Prepare input image
                test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
                test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))

                test_img_input = normalize_and_centralize_img(test_img_resize)

                # Inference
                t1 = time.time()
                stage_heatmap_np = sess.run([output_node],
                                            feed_dict={model.input_images: test_img_input})
                print('FPS: %.2f' % (1 / (time.time() - t1)))

                demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :,
                                     0:FLAGS.num_of_joints].reshape(
                    (FLAGS.heatmap_size, FLAGS.heatmap_size, FLAGS.num_of_joints))
                demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))

                vertical_imgs = []
                tmp_img = None
                joint_coord_set = np.zeros((FLAGS.num_of_joints, 2))

                for joint_num in range(FLAGS.num_of_joints):
                    # Concat until 4 img
                    if (joint_num % 4) == 0 and joint_num != 0:
                        vertical_imgs.append(tmp_img)
                        tmp_img = None

                    demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num]))

                    # Plot color joints
                    if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
                        joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]),
                                                       (FLAGS.input_size, FLAGS.input_size))
                        joint_coord_set[joint_num, :] = joint_coord
                        color_code_num = (joint_num // 4)

                        if joint_num in [0, 4, 8, 12, 16]:
                            joint_color = list(
                                map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
                            cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
                                       thickness=-1)
                        else:
                            joint_color = list(
                                map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
                            cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
                                       thickness=-1)

                    # Put text
                    tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
                    tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
                                      org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
                    tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])),
                                      org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
                    tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
                        if tmp_img is not None else tmp

                # Plot FLAGS.limbs
                for limb_num in range(len(FLAGS.limbs)):
                    if np.min(demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]]) > -2000 and np.min(
                            demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][1]]) > -2000:
                        x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0]
                        y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1]
                        x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0]
                        y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1]
                        length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
                        if length < 10000 and length > 5:
                            deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
                            polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
                                                       (int(length / 2), 3),
                                                       int(deg),
                                                       0, 360, 1)
                            color_code_num = limb_num // 4
                            limb_color = list(
                                map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num]))

                            cv2.fillConvexPoly(test_img, polygon, color=limb_color)

                if tmp_img is not None:
                    tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)),
                                         'constant', constant_values=(0, 0))
                    vertical_imgs.append(tmp_img)

                # Concat horizontally
                output_img = None
                for col in range(len(vertical_imgs)):
                    output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
                        vertical_imgs[col]

                output_img = output_img.astype(np.uint8)
                output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
                test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
                cv2.imshow('hm', output_img)
                cv2.moveWindow('hm', 2000, 200)
                cv2.imshow('rgb', test_img)
                cv2.moveWindow('rgb', 2000, 750)
                if cv2.waitKey(1) == ord('q'): break
示例#29
0
文件: eye.py 项目: pupil-labs/pupil
def eye(
    timebase,
    is_alive_flag,
    ipc_pub_url,
    ipc_sub_url,
    ipc_push_url,
    user_dir,
    version,
    eye_id,
    overwrite_cap_settings=None,
):
    """reads eye video and detects the pupil.

    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates.

    Reacts to notifications:
       ``set_detection_mapping_mode``: Sets detection method
       ``eye_process.should_stop``: Stops the eye process
       ``recording.started``: Starts recording eye video
       ``recording.stopped``: Stops recording eye video
       ``frame_publishing.started``: Starts frame publishing
       ``frame_publishing.stopped``: Stops frame publishing

    Emits notifications:
        ``eye_process.started``: Eye process started
        ``eye_process.stopped``: Eye process stopped

    Emits data:
        ``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
        ``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
    """

    # We deferr the imports becasue of multiprocessing.
    # Otherwise the world process each process also loads the other imports.
    import zmq
    import zmq_tools

    zmq_ctx = zmq.Context()
    ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
    pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
    notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))

    # logging setup
    import logging

    logging.getLogger("OpenGL").setLevel(logging.ERROR)
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.NOTSET)
    logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
    # create logger for the context of this function
    logger = logging.getLogger(__name__)

    if is_alive_flag.value:
        # indicates eye process that this is a duplicated startup
        logger.warning("Aborting redundant eye process startup")
        return

    with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
        # general imports
        import traceback
        import numpy as np
        import cv2

        # display
        import glfw
        from pyglui import ui, graph, cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
        from pyglui.cygl.utils import Named_Texture
        from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
        from gl_utils import make_coord_system_pixel_based
        from gl_utils import make_coord_system_norm_based
        from gl_utils import is_window_visible, glViewport
        from ui_roi import UIRoi

        # monitoring
        import psutil

        # helpers/utils
        from uvc import get_time_monotonic
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, timer
        from av_writer import JPEG_Writer, AV_Writer
        from ndsi import H264Writer
        from video_capture import source_classes
        from video_capture import manager_classes

        from background_helper import IPC_Logging_Task_Proxy

        IPC_Logging_Task_Proxy.push_url = ipc_push_url

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy

        pupil_detectors = {
            Detector_2D.__name__: Detector_2D,
            Detector_3D.__name__: Detector_3D,
            Detector_Dummy.__name__: Detector_Dummy,
        }

        # UI Platform tweaks
        if platform.system() == "Linux":
            scroll_factor = 10.0
            window_position_default = (600, 300 * eye_id + 30)
        elif platform.system() == "Windows":
            scroll_factor = 10.0
            window_position_default = (600, 90 + 300 * eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600, 300 * eye_id)

        icon_bar_width = 50
        window_size = None
        camera_render_size = None
        hdpi_factor = 1.0

        # g_pool holds variables for this process
        g_pool = SimpleNamespace()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = "capture"
        g_pool.process = "eye{}".format(eye_id)
        g_pool.timebase = timebase

        g_pool.ipc_pub = ipc_socket

        def get_timestamp():
            return get_time_monotonic() - g_pool.timebase.value

        g_pool.get_timestamp = get_timestamp
        g_pool.get_now = get_time_monotonic

        # Callback functions
        def on_resize(window, w, h):
            nonlocal window_size
            nonlocal camera_render_size
            nonlocal hdpi_factor

            active_window = glfw.glfwGetCurrentContext()
            glfw.glfwMakeContextCurrent(window)
            hdpi_factor = glfw.getHDPIFactor(window)
            g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
            window_size = w, h
            camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
            g_pool.gui.update_window(w, h)
            g_pool.gui.collect_menus()
            for g in g_pool.graphs:
                g.scale = hdpi_factor
                g.adjust_window_size(w, h)
            adjust_gl_view(w, h)
            glfw.glfwMakeContextCurrent(active_window)

        def on_window_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key, scancode, action, mods)

        def on_window_char(window, char):
            g_pool.gui.update_char(char)

        def on_iconify(window, iconified):
            g_pool.iconified = iconified

        def on_window_mouse_button(window, button, action, mods):
            g_pool.gui.update_button(button, action, mods)

        def on_pos(window, x, y):
            x *= hdpi_factor
            y *= hdpi_factor
            g_pool.gui.update_mouse(x, y)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x, y), camera_render_size)
                if g_pool.flip:
                    pos = 1 - pos[0], 1 - pos[1]
                pos = denormalize(pos, g_pool.capture.frame_size)
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)

        def on_scroll(window, x, y):
            g_pool.gui.update_scroll(x, y * scroll_factor)

        def on_drop(window, count, paths):
            paths = [paths[x].decode("utf-8") for x in range(count)]
            plugins = (g_pool.capture_manager, g_pool.capture)
            # call `on_drop` callbacks until a plugin indicates
            # that it has consumed the event (by returning True)
            any(p.on_drop(paths) for p in plugins)

        # load session persistent settings
        session_settings = Persistent_Dict(
            os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))
        )
        if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version:
            logger.info(
                "Session setting are from a different version of this app. I will not use those."
            )
            session_settings.clear()

        g_pool.iconified = False
        g_pool.capture = None
        g_pool.capture_manager = None
        g_pool.flip = session_settings.get("flip", False)
        g_pool.display_mode = session_settings.get("display_mode", "camera_image")
        g_pool.display_mode_info_text = {
            "camera_image": "Raw eye camera image. This uses the least amount of CPU power",
            "roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
            "algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
        }

        capture_manager_settings = session_settings.get(
            "capture_manager_settings", ("UVC_Manager", {})
        )

        manager_class_name, manager_settings = capture_manager_settings
        manager_class_by_name = {c.__name__: c for c in manager_classes}
        g_pool.capture_manager = manager_class_by_name[manager_class_name](
            g_pool, **manager_settings
        )

        if eye_id == 0:
            cap_src = ["Pupil Cam3 ID0", "Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"]
        else:
            cap_src = ["Pupil Cam3 ID1", "Pupil Cam2 ID1", "Pupil Cam1 ID1"]

        # Initialize capture
        default_settings = (
            "UVC_Source",
            {"preferred_names": cap_src, "frame_size": (320, 240), "frame_rate": 120},
        )

        capture_source_settings = overwrite_cap_settings or session_settings.get(
            "capture_settings", default_settings
        )
        source_class_name, source_settings = capture_source_settings
        source_class_by_name = {c.__name__: c for c in source_classes}
        g_pool.capture = source_class_by_name[source_class_name](
            g_pool, **source_settings
        )
        assert g_pool.capture

        g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
        roi_user_settings = session_settings.get("roi")
        if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
            g_pool.u_r.set(roi_user_settings)

        pupil_detector_settings = session_settings.get("pupil_detector_settings", None)
        last_pupil_detector = pupil_detectors[
            session_settings.get("last_pupil_detector", Detector_2D.__name__)
        ]
        g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings)

        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]

        def set_detector(new_detector):
            g_pool.pupil_detector.deinit_ui()
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_ui()

        def toggle_general_settings(collapsed):
            # this is the menu toggle logic.
            # Only one menu can be open.
            # If no menu is open the menubar should collapse.
            g_pool.menubar.collapsed = collapsed
            for m in g_pool.menubar.elements:
                m.collapsed = True
            general_settings.collapsed = collapsed

        # Initialize glfw
        glfw.glfwInit()
        title = "Pupil Capture - eye {}".format(eye_id)

        width, height = g_pool.capture.frame_size
        width *= 2
        height *= 2
        width += icon_bar_width
        width, height = session_settings.get("window_size", (width, height))

        main_window = glfw.glfwCreateWindow(width, height, title, None, None)
        window_pos = session_settings.get("window_position", window_position_default)
        glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui_user_scale = new_scale
            on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125)

        # setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
        g_pool.menubar = ui.Scrolling_Menu(
            "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
        )
        g_pool.iconbar = ui.Scrolling_Menu(
            "Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
        )
        g_pool.gui.append(g_pool.menubar)
        g_pool.gui.append(g_pool.iconbar)

        general_settings = ui.Growing_Menu("General", header_pos="headline")
        general_settings.append(
            ui.Selector(
                "gui_user_scale",
                g_pool,
                setter=set_scale,
                selection=[0.8, 0.9, 1.0, 1.1, 1.2],
                label="Interface Size",
            )
        )

        def set_window_size():
            f_width, f_height = g_pool.capture.frame_size
            f_width *= 2
            f_height *= 2
            f_width += int(icon_bar_width * g_pool.gui.scale)
            glfw.glfwSetWindowSize(main_window, f_width, f_height)

        def uroi_on_mouse_button(button, action, mods):
            if g_pool.display_mode == "roi":
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    # if the roi interacts we dont want
                    # the gui to interact as well
                    return
                elif action == glfw.GLFW_PRESS:
                    x, y = glfw.glfwGetCursorPos(main_window)
                    # pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
                    x *= hdpi_factor
                    y *= hdpi_factor
                    pos = normalize((x, y), camera_render_size)
                    if g_pool.flip:
                        pos = 1 - pos[0], 1 - pos[1]
                    # Position in img pixels
                    pos = denormalize(
                        pos, g_pool.capture.frame_size
                    )  # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(
                        pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size
                    ):
                        # if the roi interacts we dont want
                        # the gui to interact as well
                        return

        general_settings.append(ui.Button("Reset window size", set_window_size))
        general_settings.append(ui.Switch("flip", g_pool, label="Flip image display"))
        general_settings.append(
            ui.Selector(
                "display_mode",
                g_pool,
                setter=set_display_mode_info,
                selection=["camera_image", "roi", "algorithm"],
                labels=["Camera Image", "ROI", "Algorithm"],
                label="Mode",
            )
        )
        g_pool.display_mode_info = ui.Info_Text(
            g_pool.display_mode_info_text[g_pool.display_mode]
        )

        general_settings.append(g_pool.display_mode_info)

        detector_selector = ui.Selector(
            "pupil_detector",
            getter=lambda: g_pool.pupil_detector.__class__,
            setter=set_detector,
            selection=[Detector_Dummy, Detector_2D, Detector_3D],
            labels=["disabled", "C++ 2d detector", "C++ 3d detector"],
            label="Detection method",
        )
        general_settings.append(detector_selector)

        g_pool.menubar.append(general_settings)
        icon = ui.Icon(
            "collapsed",
            general_settings,
            label=chr(0xE8B8),
            on_val=False,
            off_val=True,
            setter=toggle_general_settings,
            label_font="pupil_icons",
        )
        icon.tooltip = "General Settings"
        g_pool.iconbar.append(icon)
        toggle_general_settings(False)

        g_pool.pupil_detector.init_ui()
        g_pool.capture.init_ui()
        g_pool.capture_manager.init_ui()
        g_pool.writer = None

        def replace_source(source_class_name, source_settings):
            g_pool.capture.deinit_ui()
            g_pool.capture.cleanup()
            g_pool.capture = source_class_by_name[source_class_name](
                g_pool, **source_settings
            )
            g_pool.capture.init_ui()
            if g_pool.writer:
                logger.info("Done recording.")
                try:
                    g_pool.writer.release()
                except RuntimeError:
                    logger.error("No eye video recorded")
                g_pool.writer = None

        g_pool.replace_source = replace_source  # for ndsi capture

        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
        glfw.glfwSetKeyCallback(main_window, on_window_key)
        glfw.glfwSetCharCallback(main_window, on_window_char)
        glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
        glfw.glfwSetCursorPosCallback(main_window, on_pos)
        glfw.glfwSetScrollCallback(main_window, on_scroll)
        glfw.glfwSetDropCallback(main_window, on_drop)

        # load last gui configuration
        g_pool.gui.configuration = session_settings.get("ui_config", {})

        # set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = g_pool.get_timestamp()

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20, 50)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = "CPU %0.1f"

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140, 50)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"
        g_pool.graphs = [cpu_graph, fps_graph]

        # set the last saved window size
        on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))

        should_publish_frames = False
        frame_publish_format = "jpeg"
        frame_publish_format_recent_warning = False

        # create a timer to control window update frequency
        window_update_timer = timer(1 / 60)

        def window_should_update():
            return next(window_update_timer)

        logger.warning("Process started.")

        frame = None

        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if notify_sub.new_data:
                t, notification = notify_sub.recv()
                subject = notification["subject"]
                if subject.startswith("eye_process.should_stop"):
                    if notification["eye_id"] == eye_id:
                        break
                elif subject == "set_detection_mapping_mode":
                    if notification["mode"] == "3d":
                        if not isinstance(g_pool.pupil_detector, Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only = True
                    elif notification["mode"] == "2d":
                        if not isinstance(g_pool.pupil_detector, Detector_2D):
                            set_detector(Detector_2D)
                        detector_selector.read_only = False
                    else:
                        if not isinstance(g_pool.pupil_detector, Detector_Dummy):
                            set_detector(Detector_Dummy)
                        detector_selector.read_only = True
                elif subject == "recording.started":
                    if notification["record_eye"] and g_pool.capture.online:
                        record_path = notification["rec_path"]
                        raw_mode = notification["compression"]
                        logger.info("Will save eye video to: {}".format(record_path))
                        video_path = os.path.join(
                            record_path, "eye{}.mp4".format(eye_id)
                        )
                        if raw_mode and frame and g_pool.capture.jpeg_support:
                            g_pool.writer = JPEG_Writer(
                                video_path, g_pool.capture.frame_rate
                            )
                        elif hasattr(g_pool.capture._recent_frame, "h264_buffer"):
                            g_pool.writer = H264Writer(
                                video_path,
                                g_pool.capture.frame_size[0],
                                g_pool.capture.frame_size[1],
                                g_pool.capture.frame_rate,
                            )
                        else:
                            g_pool.writer = AV_Writer(
                                video_path, g_pool.capture.frame_rate
                            )
                elif subject == "recording.stopped":
                    if g_pool.writer:
                        logger.info("Done recording.")
                        try:
                            g_pool.writer.release()
                        except RuntimeError:
                            logger.error("No eye video recorded")
                        g_pool.writer = None
                elif subject.startswith("meta.should_doc"):
                    ipc_socket.notify(
                        {
                            "subject": "meta.doc",
                            "actor": "eye{}".format(eye_id),
                            "doc": eye.__doc__,
                        }
                    )
                elif subject.startswith("frame_publishing.started"):
                    should_publish_frames = True
                    frame_publish_format = notification.get("format", "jpeg")
                elif subject.startswith("frame_publishing.stopped"):
                    should_publish_frames = False
                    frame_publish_format = "jpeg"
                elif (
                    subject.startswith("start_eye_capture")
                    and notification["target"] == g_pool.process
                ):
                    replace_source(notification["name"], notification["args"])
                elif notification["subject"].startswith("pupil_detector.set_property"):
                    target_process = notification.get("target", g_pool.process)
                    should_apply = target_process == g_pool.process

                    if should_apply:
                        try:
                            property_name = notification["name"]
                            property_value = notification["value"]
                            if "2d" in notification["subject"]:
                                g_pool.pupil_detector.set_2d_detector_property(
                                    property_name, property_value
                                )
                            elif "3d" in notification["subject"]:
                                if not isinstance(g_pool.pupil_detector, Detector_3D):
                                    raise ValueError(
                                        "3d properties are only available"
                                        " if 3d detector is active"
                                    )
                                g_pool.pupil_detector.set_3d_detector_property(
                                    property_name, property_value
                                )
                            else:
                                raise KeyError(
                                    "Notification subject does not "
                                    "specifiy detector type."
                                )
                            logger.debug(
                                "`{}` property set to {}".format(
                                    property_name, property_value
                                )
                            )
                        except KeyError:
                            logger.error("Malformed notification received")
                            logger.debug(traceback.format_exc())
                        except (ValueError, TypeError):
                            logger.error("Invalid property or value")
                            logger.debug(traceback.format_exc())
                elif notification["subject"].startswith(
                    "pupil_detector.broadcast_properties"
                ):
                    target_process = notification.get("target", g_pool.process)
                    should_respond = target_process == g_pool.process
                    if should_respond:
                        props = g_pool.pupil_detector.get_detector_properties()
                        properties_broadcast = {
                            "subject": "pupil_detector.properties.{}".format(eye_id),
                            **props,  # add properties to broadcast
                        }
                        ipc_socket.notify(properties_broadcast)
                g_pool.capture.on_notify(notification)
                g_pool.capture_manager.on_notify(notification)

            # Get an image from the grabber
            event = {}
            g_pool.capture.recent_events(event)
            frame = event.get("frame")
            g_pool.capture_manager.recent_events(event)
            if frame:
                f_width, f_height = g_pool.capture.frame_size
                if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
                    f_height,
                    f_width,
                ):
                    g_pool.pupil_detector.on_resolution_change(
                        (g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
                        g_pool.capture.frame_size,
                    )
                    g_pool.u_r = UIRoi((f_height, f_width))
                if should_publish_frames:
                    try:
                        if frame_publish_format == "jpeg":
                            data = frame.jpeg_buffer
                        elif frame_publish_format == "yuv":
                            data = frame.yuv_buffer
                        elif frame_publish_format == "bgr":
                            data = frame.bgr
                        elif frame_publish_format == "gray":
                            data = frame.gray
                        assert data is not None
                    except (AttributeError, AssertionError, NameError):
                        if not frame_publish_format_recent_warning:
                            frame_publish_format_recent_warning = True
                            logger.warning(
                                '{}s are not compatible with format "{}"'.format(
                                    type(frame), frame_publish_format
                                )
                            )
                    else:
                        frame_publish_format_recent_warning = False
                        pupil_socket.send(
                            {
                                "topic": "frame.eye.{}".format(eye_id),
                                "width": frame.width,
                                "height": frame.height,
                                "index": frame.index,
                                "timestamp": frame.timestamp,
                                "format": frame_publish_format,
                                "__raw_data__": [data],
                            }
                        )

                t = frame.timestamp
                dt, ts = t - ts, t
                try:
                    fps_graph.add(1.0 / dt)
                except ZeroDivisionError:
                    pass

                if g_pool.writer:
                    g_pool.writer.write_video_frame(frame)

                # pupil ellipse detection
                result = g_pool.pupil_detector.detect(
                    frame, g_pool.u_r, g_pool.display_mode == "algorithm"
                )
                if result is not None:
                    result["id"] = eye_id
                    result["topic"] = "pupil.{}".format(eye_id)
                    pupil_socket.send(result)

            cpu_graph.update()

            # GL drawing
            if window_should_update():
                if is_window_visible(main_window):
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    if frame:
                        # switch to work in normalized coordinate space
                        if g_pool.display_mode == "algorithm":
                            g_pool.image_tex.update_from_ndarray(frame.img)
                        elif g_pool.display_mode in ("camera_image", "roi"):
                            g_pool.image_tex.update_from_ndarray(frame.gray)
                        else:
                            pass
                    glViewport(0, 0, *camera_render_size)
                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    f_width, f_height = g_pool.capture.frame_size
                    make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
                    if frame and result:
                        if result["method"] == "3d c++":
                            eye_ball = result["projected_sphere"]
                            try:
                                pts = cv2.ellipse2Poly(
                                    (
                                        int(eye_ball["center"][0]),
                                        int(eye_ball["center"][1]),
                                    ),
                                    (
                                        int(eye_ball["axes"][0] / 2),
                                        int(eye_ball["axes"][1] / 2),
                                    ),
                                    int(eye_ball["angle"]),
                                    0,
                                    360,
                                    8,
                                )
                            except ValueError as e:
                                pass
                            else:
                                draw_polyline(
                                    pts,
                                    2,
                                    RGBA(0.0, 0.9, 0.1, result["model_confidence"]),
                                )
                        if result["confidence"] > 0:
                            if "ellipse" in result:
                                pts = cv2.ellipse2Poly(
                                    (
                                        int(result["ellipse"]["center"][0]),
                                        int(result["ellipse"]["center"][1]),
                                    ),
                                    (
                                        int(result["ellipse"]["axes"][0] / 2),
                                        int(result["ellipse"]["axes"][1] / 2),
                                    ),
                                    int(result["ellipse"]["angle"]),
                                    0,
                                    360,
                                    15,
                                )
                                confidence = result["confidence"] * 0.7
                                draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence))
                                draw_points(
                                    [result["ellipse"]["center"]],
                                    size=20,
                                    color=RGBA(1.0, 0.0, 0.0, confidence),
                                    sharpness=1.0,
                                )

                    glViewport(0, 0, *camera_render_size)
                    make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
                    # render the ROI
                    g_pool.u_r.draw(g_pool.gui.scale)
                    if g_pool.display_mode == "roi":
                        g_pool.u_r.draw_points(g_pool.gui.scale)

                    glViewport(0, 0, *window_size)
                    make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)
                    # render graphs
                    fps_graph.draw()
                    cpu_graph.draw()

                    # render GUI
                    unused_elements = g_pool.gui.update()
                    for butt in unused_elements.buttons:
                        uroi_on_mouse_button(*butt)

                    make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)

                    g_pool.pupil_detector.visualize()  # detector decides if we visualize or not

                    # update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()

        # END while running

        # in case eye recording was still runnnig: Save&close
        if g_pool.writer:
            logger.info("Done recording eye.")
            g_pool.writer = None

        glfw.glfwRestoreWindow(main_window)  # need to do this for windows os
        # save session persistent settings
        session_settings["gui_scale"] = g_pool.gui_user_scale
        session_settings["roi"] = g_pool.u_r.get()
        session_settings["flip"] = g_pool.flip
        session_settings["display_mode"] = g_pool.display_mode
        session_settings["ui_config"] = g_pool.gui.configuration
        session_settings["capture_settings"] = (
            g_pool.capture.class_name,
            g_pool.capture.get_init_dict(),
        )
        session_settings["capture_manager_settings"] = (
            g_pool.capture_manager.class_name,
            g_pool.capture_manager.get_init_dict(),
        )
        session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
        session_settings["version"] = str(g_pool.version)
        session_settings[
            "last_pupil_detector"
        ] = g_pool.pupil_detector.__class__.__name__
        session_settings[
            "pupil_detector_settings"
        ] = g_pool.pupil_detector.get_settings()

        session_window_size = glfw.glfwGetWindowSize(main_window)
        if 0 not in session_window_size:
            session_settings["window_size"] = session_window_size

        session_settings.close()

        g_pool.capture.deinit_ui()
        g_pool.capture_manager.deinit_ui()
        g_pool.pupil_detector.deinit_ui()

        g_pool.pupil_detector.cleanup()
        g_pool.capture_manager.cleanup()
        g_pool.capture.cleanup()

        glfw.glfwDestroyWindow(main_window)
        g_pool.gui.terminate()
        glfw.glfwTerminate()
        logger.info("Process shutting down.")
 def gl_display(self):
     if self.fixation is not None:
         abs_fixation = denormalize(self.fixation,self.g_pool.capture.frame_size, flip_y=True)
         ellipse = cv2.ellipse2Poly((int(abs_fixation[0]), int(abs_fixation[1])),(25, 25),0,0,360,15)
         draw_gl_polyline(ellipse,(0.,0.,.5,.75),'Polygon')
示例#31
0
def main():

	param, model = config_reader()
	model_xml = "/home/yue/Realtime_Multi-Person_Pose_Estimation/model/_trained_COCO/pose_iter_440000.xml"
	model_bin = os.path.splitext(model_xml)[0] + ".bin"
	prob_threshold = 0.5
	labels_map = None

	print("Initializing plugin for CPU device...")
	plugin = IEPlugin(device="CPU", plugin_dirs=None)

	print("Adding CPU extenstions...")
	plugin.add_cpu_extension("/opt/intel/computer_vision_sdk/deployment_tools/inference_engine/lib/ubuntu_16.04/intel64/libcpu_extension_sse4.so")

	print("Reading IR...")
	net = IENetwork.from_ir(model=model_xml, weights=model_bin)
	input_blob = next(iter(net.inputs))
	out_blob = next(iter(net.outputs))

	print("Loading IR to the plugin...")
	exec_net = plugin.load(network=net, num_requests=2)
	# Read and pre-process input image
	n, c, h, w = net.inputs[input_blob]
	print((n,c,h,w))

	input_stream = "../sample_video/dance.mp4"
	cap = cv.VideoCapture(input_stream)
	cur_request_id = 0
	next_request_id = 1

	if (cap.isOpened()== False): 
	  print("Error opening video stream or file")
		
	print("Starting inference in async mode...")
	is_async_mode = True
	render_time = 0


	# find connection in the specified sequence, center 29 is in the position 15
	limbSeq = [[2,3], [2,6], [3,4], [4,5], [6,7], [7,8], [2,9], [9,10], \
			   [10,11], [2,12], [12,13], [13,14], [2,1], [1,15], [15,17], \
			   [1,16], [16,18], [3,17], [6,18]]
	# the middle joints heatmap correpondence
	mapIdx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], \
			  [23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], \
			  [55,56], [37,38], [45,46]]
	colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
			  [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
			  [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
	fps = 0
	
	while cap.isOpened():
		ret, frame = cap.read()
		if not ret:
			break
		
		initial_w = cap.get(3)
		initial_h = cap.get(4)
		
		image = cv.resize(frame, (w, h))

		image = image.transpose((2, 0, 1))  # Change data layout from HWC to CHW
		image = image.reshape((1,3,368,368))/256 - 0.5
		img_show = frame;
		# Main sync point:
		# in the truly Async mode we start the NEXT infer request, while waiting for the CURRENT to complete
		# in the regular mode we start the CURRENT request and immediately wait for it's completion
		inf_start = time.time()
		
		res = exec_net.infer(inputs={input_blob: image})
		
		inf_end = time.time()
		det_time = inf_end - inf_start

		# Parse detection results of the current request
		res_heatMap = res['Mconv7_stage6_L2']
		res_paf = res['Mconv7_stage6_L1']

		#Process outputs
		res_heatMap = np.squeeze(res_heatMap, axis=0)
		res_paf = np.squeeze(res_paf, axis=0)

		# extract outputs, resize, and remove padding
		heatmap = np.transpose(res_heatMap, (1,2,0)) # output 1 is heatmaps

		heatmap = cv.resize(heatmap, (frame.shape[1], frame.shape[0]), interpolation=cv.INTER_CUBIC)

		paf = np.transpose(res_paf, (1,2,0)) # output 0 is PAFs

		paf = cv.resize(paf, (frame.shape[1], frame.shape[0]), interpolation=cv.INTER_CUBIC)

		heatmap_avg = heatmap 
		paf_avg = paf 


		U = paf_avg[:,:,16] * -1
		V = paf_avg[:,:,17]
		X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0]))
		M = np.zeros(U.shape, dtype='bool')
		M[U**2 + V**2 < 0.5 * 0.5] = True
		U = ma.masked_array(U, mask=M)
		V = ma.masked_array(V, mask=M)

		all_peaks = []
		peak_counter = 0

		for part in range(19-1):
			x_list = []
			y_list = []
			map_ori = heatmap_avg[:,:,part]
			map = gaussian_filter(map_ori, sigma=3)

			map_left = np.zeros(map.shape)
			map_left[1:,:] = map[:-1,:]
			map_right = np.zeros(map.shape)
			map_right[:-1,:] = map[1:,:]
			map_up = np.zeros(map.shape)
			map_up[:,1:] = map[:,:-1]
			map_down = np.zeros(map.shape)
			map_down[:,:-1] = map[:,1:]

			peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param['thre1']))
			peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
			peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks]
			id = range(peak_counter, peak_counter + len(peaks))
			peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))]

			all_peaks.append(peaks_with_score_and_id)
			peak_counter += len(peaks)


		connection_all = []
		special_k = []
		mid_num = 10

		for k in range(len(mapIdx)):
			score_mid = paf_avg[:,:,[x-19 for x in mapIdx[k]]]
			candA = all_peaks[limbSeq[k][0]-1]
			candB = all_peaks[limbSeq[k][1]-1]
			nA = len(candA)
			nB = len(candB)
			indexA, indexB = limbSeq[k]
			if(nA != 0 and nB != 0):
				connection_candidate = []
				for i in range(nA):
					for j in range(nB):
						vec = np.subtract(candB[j][:2], candA[i][:2])
						norm = math.sqrt(vec[0]*vec[0] + vec[1]*vec[1])
						vec = np.divide(vec, norm)

						startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
									   np.linspace(candA[i][1], candB[j][1], num=mid_num)))

						vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
										  for I in range(len(startend))])
						vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
										  for I in range(len(startend))])
						score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
						score_with_dist_prior = sum(score_midpts)/len(score_midpts) + min(0.5*frame.shape[0]/norm-1, 0)
						criterion1 = len(np.nonzero(score_midpts > param['thre2'])[0]) > 0.8 * len(score_midpts)
						criterion2 = score_with_dist_prior > 0
						if criterion1 and criterion2:
							connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior+candA[i][2]+candB[j][2]])

				connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
				connection = np.zeros((0,5))
				for c in range(len(connection_candidate)):
					i,j,s = connection_candidate[c][0:3]
					if(i not in connection[:,3] and j not in connection[:,4]):
						connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
						if(len(connection) >= min(nA, nB)):
							break

				connection_all.append(connection)
			else:
				special_k.append(k)
				connection_all.append([])
		# last number in each row is the total parts number of that person
		# the second last number in each row is the score of the overall configuration
		subset = -1 * np.ones((0, 20))
		candidate = np.array([item for sublist in all_peaks for item in sublist])

		for k in range(len(mapIdx)):
			if k not in special_k:
				partAs = connection_all[k][:,0]
				partBs = connection_all[k][:,1]
				indexA, indexB = np.array(limbSeq[k]) - 1

				for i in range(len(connection_all[k])): #= 1:size(temp,1)
					found = 0
					subset_idx = [-1, -1]
					for j in range(len(subset)): #1:size(subset,1):
						if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
							subset_idx[found] = j
							found += 1

					if found == 1:
						j = subset_idx[0]
						if(subset[j][indexB] != partBs[i]):
							subset[j][indexB] = partBs[i]
							subset[j][-1] += 1
							subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
					elif found == 2: # if found 2 and disjoint, merge them
						j1, j2 = subset_idx
						print ("found = 2")
						membership = ((subset[j1]>=0).astype(int) + (subset[j2]>=0).astype(int))[:-2]
						if len(np.nonzero(membership == 2)[0]) == 0: #merge
							subset[j1][:-2] += (subset[j2][:-2] + 1)
							subset[j1][-2:] += subset[j2][-2:]
							subset[j1][-2] += connection_all[k][i][2]
							subset = np.delete(subset, j2, 0)
						else: # as like found == 1
							subset[j1][indexB] = partBs[i]
							subset[j1][-1] += 1
							subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]

					# if find no partA in the subset, create a new subset
					elif not found and k < 17:
						row = -1 * np.ones(20)
						row[indexA] = partAs[i]
						row[indexB] = partBs[i]
						row[-1] = 2
						row[-2] = sum(candidate[connection_all[k][i,:2].astype(int), 2]) + connection_all[k][i][2]
						subset = np.vstack([subset, row])
		deleteIdx = [];
		for i in range(len(subset)):
			if subset[i][-1] < 4 or subset[i][-2]/subset[i][-1] < 0.4:
				deleteIdx.append(i)
		subset = np.delete(subset, deleteIdx, axis=0)


		stickwidth = 2
		cmap = matplotlib.cm.get_cmap('hsv')

		for i in range(18):
			rgba = np.array(cmap(1 - i/18. - 1./36))
			rgba[0:3] *= 255
			for j in range(len(all_peaks[i])):
				cv.circle(img_show, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)

		to_plot = cv.addWeighted(frame, 0.3, img_show, 0.7, 0)

		for i in range(17):
			for n in range(len(subset)):
				index = subset[n][np.array(limbSeq[i])-1]
				if -1 in index:
					continue
				cur_canvas = img_show.copy()
				Y = candidate[index.astype(int), 0]
				X = candidate[index.astype(int), 1]
				mX = np.mean(X)
				mY = np.mean(Y)
				length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
				angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
				polygon = cv.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
				cv.fillConvexPoly(cur_canvas, polygon, colors[i])
				img_show = cv.addWeighted(img_show, 0.4, cur_canvas, 0.6, 0)


		#################################################################################################################
		# Draw performance stats

		inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \
						   "Inference time: {:.3f} ms".format(det_time * 1000)
		render_time_message = "OpenCV rendering time: {:.3f} ms".format(render_time * 1000)
		async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \
							 "Async mode is off. Processing request {}".format(cur_request_id)
		fps_message = "FPS: {:.1f}".format(fps);

		cv.putText(img_show, inf_time_message, (15, 15), cv.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
		cv.putText(img_show, render_time_message, (15, 30), cv.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
		cv.putText(img_show, fps_message, (15, 45), cv.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
		
		render_start = time.time()
		cv.imshow("Detection Results", img_show)
		render_end = time.time()
		render_time = render_end - render_start
		fps = float(60/(render_time*1000))
		
		key = cv.waitKey(1)
		if key == 27:
			break
		if (9 == key):
			is_async_mode = not is_async_mode
			print("Switched to {} mode".format("async" if is_async_mode else "sync"))

		if is_async_mode:
			cur_request_id, next_request_id = next_request_id, cur_request_id
			
	cv.destroyAllWindows()
	del exec_net
	del plugin
示例#32
0
def process(model, input_path):

    origin_img = cv2.imread(input_path)
    normed_img = normalize(origin_img)

    height, width, _ = normed_img.shape

    multiplier = [x * boxsize / height for x in scale_search]

    heatmap_avg = np.zeros((height, width, 19))  # num_point
    paf_avg = np.zeros((height, width, 38))  # num_vector

    for m in range(len(multiplier)):
        scale = multiplier[m]

        # preprocess
        imgToTest = cv2.resize(normed_img, (0, 0),
                               fx=scale,
                               fy=scale,
                               interpolation=cv2.INTER_CUBIC)
        imgToTest_padded, pad = padRightDownCorner(imgToTest, stride, padValue)

        input_img = np.transpose(imgToTest_padded[:, :, :, np.newaxis],
                                 (3, 2, 0, 1))  # required shape (1, c, h, w)
        mask = np.ones(
            (1, 1, input_img.shape[2] / stride, input_img.shape[3] / stride),
            dtype=np.float32)

        input_var = torch.autograd.Variable(torch.from_numpy(input_img).cuda())
        mask_var = torch.autograd.Variable(torch.from_numpy(mask).cuda())

        # get the features
        vec1, heat1, vec2, heat2, vec3, heat3, vec4, heat4, vec5, heat5, vec6, heat6 = model(
            input_var, mask_var)

        # get the heatmap
        heatmap = heat6.data.cpu().numpy()
        heatmap = np.transpose(np.squeeze(heatmap), (1, 2, 0))  # (h, w, c)
        heatmap = cv2.resize(heatmap, (0, 0),
                             fx=stride,
                             fy=stride,
                             interpolation=cv2.INTER_CUBIC)
        heatmap = heatmap[:imgToTest_padded.shape[0] -
                          pad[2], :imgToTest_padded.shape[1] - pad[3], :]
        heatmap = cv2.resize(heatmap, (width, height),
                             interpolation=cv2.INTER_CUBIC)
        heatmap_avg = heatmap_avg + heatmap / len(multiplier)

        # get the paf
        paf = vec6.data.cpu().numpy()
        paf = np.transpose(np.squeeze(paf), (1, 2, 0))  # (h, w, c)
        paf = cv2.resize(paf, (0, 0),
                         fx=stride,
                         fy=stride,
                         interpolation=cv2.INTER_CUBIC)
        paf = paf[:imgToTest_padded.shape[0] -
                  pad[2], :imgToTest_padded.shape[1] - pad[3], :]
        paf = cv2.resize(paf, (width, height), interpolation=cv2.INTER_CUBIC)
        paf_avg = paf_avg + paf / len(multiplier)

    all_peaks = []  # all of the possible points by classes.
    peak_counter = 0

    for part in range(1, 19):
        map_ori = heatmap_avg[:, :, part]
        map = gaussian_filter(map_ori, sigma=3)

        map_left = np.zeros(map.shape)
        map_left[:, 1:] = map[:, :-1]
        map_right = np.zeros(map.shape)
        map_right[:, :-1] = map[:, 1:]
        map_up = np.zeros(map.shape)
        map_up[1:, :] = map[:-1, :]
        map_down = np.zeros(map.shape)
        map_down[:-1, :] = map[1:, :]

        # get the salient point and its score > thre_point
        peaks_binary = np.logical_and.reduce(
            (map >= map_left, map >= map_right, map >= map_up, map >= map_down,
             map > thre_point))
        peaks = list(
            zip(np.nonzero(peaks_binary)[1],
                np.nonzero(peaks_binary)[0]))  # (w, h)

        # a point format: (w, h, score, number)
        peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
        id = range(peak_counter, peak_counter + len(peaks))
        peaks_with_score_and_id = [
            peaks_with_score[i] + (id[i], ) for i in range(len(id))
        ]

        all_peaks.append(peaks_with_score_and_id)
        peak_counter += len(peaks)

    connection_all = []  # save all of the possible lines by classes.
    special_k = []  # save the lines, which haven't legal points.
    mid_num = 10  # could adjust to accelerate (small) or improve accuracy(large).

    for k in range(len(mapIdx)):

        score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]]
        candA = all_peaks[limbSeq[k][0] - 1]
        candB = all_peaks[limbSeq[k][1] - 1]

        lenA = len(candA)
        lenB = len(candB)

        if lenA != 0 and lenB != 0:
            connection_candidate = []
            for i in range(lenA):
                for j in range(lenB):
                    vec = np.subtract(candB[j][:2],
                                      candA[i][:2])  # the vector of BA
                    norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
                    if norm == 0:
                        continue
                    vec = np.divide(vec, norm)

                    startend = list(
                        zip(np.linspace(candA[i][0], candB[j][0], num=mid_num),
                            np.linspace(candA[i][1], candB[j][1],
                                        num=mid_num)))

                    # get the vector between A and B.
                    vec_x = np.array([
                        score_mid[int(round(startend[I][1])),
                                  int(round(startend[I][0])), 0]
                        for I in range(len(startend))
                    ])
                    vec_y = np.array([
                        score_mid[int(round(startend[I][1])),
                                  int(round(startend[I][0])), 1]
                        for I in range(len(startend))
                    ])

                    score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(
                        vec_y, vec[1])
                    score_with_dist_prior = sum(score_midpts) / len(
                        score_midpts) + min(0.5 * height / norm - 1, 0)  # ???
                    criterion1 = len(np.nonzero(
                        score_midpts > thre_line)[0]) > 0.8 * len(score_midpts)
                    criterion2 = score_with_dist_prior > 0
                    if criterion1 and criterion2:
                        connection_candidate.append([
                            i, j, score_with_dist_prior,
                            score_with_dist_prior + candA[i][2], candB[j][2]
                        ])

            # sort the possible line from large to small order.
            connection_candidate = sorted(connection_candidate,
                                          key=lambda x: x[2],
                                          reverse=True)
            connection = np.zeros((0, 5))

            for c in range(len(connection_candidate)):
                i, j, s = connection_candidate[c][0:3]
                if (i not in connection[:, 3] and j not in connection[:, 4]):
                    # the number of A point, the number of B point, score, A point, B point
                    connection = np.vstack(
                        [connection, [candA[i][3], candB[j][3], s, i, j]])
                    if len(connection) >= min(lenA, lenB):
                        break
            connection_all.append(connection)
        else:
            special_k.append(k)
            connection_all.append([])

    subset = -1 * np.ones((0, 20))
    candidate = np.array([item for sublist in all_peaks for item in sublist])

    for k in range(len(mapIdx)):
        if k not in special_k:
            partAs = connection_all[k][:, 0]
            partBs = connection_all[k][:, 1]
            indexA, indexB = np.array(limbSeq[k]) - 1

            for i in range(len(connection_all[k])):
                found = 0
                flag = [False, False]
                subset_idx = [-1, -1]
                for j in range(len(subset)):
                    # fix the bug, found == 2 and not joint will lead someone occur more than once.
                    # if more than one, we choose the subset, which has a higher score.
                    if subset[j][indexA] == partAs[i]:
                        if flag[0] == False:
                            flag[0] = found
                            subset_idx[found] = j
                            flag[0] = True
                            found += 1
                        else:
                            ids = subset_idx[flag[0]]
                            if subset[ids][-1] < subset[j][-1]:
                                subset_idx[flag[0]] = j
                    if subset[j][indexB] == partBs[i]:
                        if flag[1] == False:
                            flag[1] = found
                            subset_idx[found] = j
                            flag[1] = True
                            found += 1
                        else:
                            ids = subset_idx[flag[1]]
                            if subset[ids][-1] < subset[j][-1]:
                                subset_idx[flag[1]] = j

                if found == 1:
                    j = subset_idx[0]
                    if (subset[j][indexB] != partBs[i]):
                        subset[j][indexB] = partBs[i]
                        subset[j][-1] += 1
                        subset[j][-2] += candidate[partBs[i].astype(int),
                                                   2] + connection_all[k][i][2]
                elif found == 2:  # if found equals to 2 and disjoint, merge them
                    j1, j2 = subset_idx
                    membership = ((subset[j1] >= 0).astype(int) +
                                  (subset[j2] >= 0).astype(int))[:-2]
                    if len(np.nonzero(membership == 2)[0]) == 0:  # merge
                        subset[j1][:-2] += (subset[j2][:-2] + 1)
                        subset[j1][-2:] += subset[j2][-2:]
                        subset[j1][-2] += connection_all[k][i][2]
                        subset = np.delete(subset, j2, 0)
                    else:  # as like found == 1
                        subset[j1][indexB] = partBs[i]
                        subset[j1][-1] += 1
                        subset[j1][-2] += candidate[
                            partBs[i].astype(int), 2] + connection_all[k][i][2]
                elif not found and k < 17:
                    row = -1 * np.ones(20)
                    row[indexA] = partAs[i]
                    row[indexB] = partBs[i]
                    row[-1] = 2
                    row[-2] = sum(
                        candidate[connection_all[k][i, :2].astype(int),
                                  2]) + connection_all[k][i][2]
                    subset = np.vstack([subset, row])

    # delete som rows of subset which has few parts occur
    deleteIdx = []
    for i in range(len(subset)):
        if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
            deleteIdx.append(i)
    subset = np.delete(subset, deleteIdx, axis=0)

    # draw points
    canvas = cv2.imread(input_path)
    for i in range(18):
        for j in range(len(all_peaks[i])):
            cv2.circle(canvas,
                       all_peaks[i][j][0:2],
                       4,
                       colors[i],
                       thickness=-1)

    # draw lines
    for i in range(17):
        for n in range(len(subset)):
            index = subset[n][np.array(limbSeq[i]) - 1]
            if -1 in index:
                continue
            cur_canvas = canvas.copy()
            Y = candidate[index.astype(int), 0]
            X = candidate[index.astype(int), 1]
            mX = np.mean(X)
            mY = np.mean(Y)
            length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5
            angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
            polygon = cv2.ellipse2Poly(
                (int(mY), int(mX)), (int(length / 2), stickwidth), int(angle),
                0, 360, 1)
            cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
            canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)

    return canvas
示例#33
0
def main(argv):
    tf_device = '/gpu:0'
    with tf.device(tf_device):
        """Build graph
        """
        if FLAGS.color_channel == 'RGB':
            input_data = tf.placeholder(
                dtype=tf.float32,
                shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
                name='input_image')
        else:
            input_data = tf.placeholder(
                dtype=tf.float32,
                shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
                name='input_image')

        center_map = tf.placeholder(
            dtype=tf.float32,
            shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
            name='center_map')

        model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
        model.build_model(input_data, center_map, 1)

    saver = tf.train.Saver()
    """Create session and restore weights
    """
    sess = tf.Session()

    sess.run(tf.global_variables_initializer())
    if FLAGS.model_path.endswith('pkl'):
        model.load_weights_from_file(FLAGS.model_path, sess, False)
    else:
        saver.restore(sess, FLAGS.model_path)

    test_center_map = cpm_utils.gaussian_img(FLAGS.input_size,
                                             FLAGS.input_size,
                                             FLAGS.input_size / 2,
                                             FLAGS.input_size / 2,
                                             FLAGS.cmap_radius)
    test_center_map = np.reshape(test_center_map,
                                 [1, FLAGS.input_size, FLAGS.input_size, 1])

    # Check weights
    for variable in tf.trainable_variables():
        with tf.variable_scope('', reuse=True):
            var = tf.get_variable(variable.name.split(':0')[0])
            print(variable.name, np.mean(sess.run(var)))

    # Create kalman filters
    if FLAGS.KALMAN_ON:
        kalman_filter_array = [
            cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)
        ]
        for _, joint_kalman_filter in enumerate(kalman_filter_array):
            joint_kalman_filter.transitionMatrix = np.array(
                [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
                np.float32)
            joint_kalman_filter.measurementMatrix = np.array(
                [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
            joint_kalman_filter.processNoiseCov = np.array(
                [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
                np.float32) * FLAGS.kalman_noise
    else:
        kalman_filter_array = None

    # read in video / flow frames
    if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
        # OpenCV can only read in '.avi' files
        cam = imageio.get_reader(FLAGS.DEMO_TYPE)
    else:
        cam = cv2.VideoCapture(FLAGS.cam_num)

    # iamge processing
    with tf.device(tf_device):
        if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
            ori_fps = cam.get_meta_data()['fps']
            print('This video fps is %f' % ori_fps)
            video_length = cam.get_length()
            writer_path = os.path.join('results',
                                       os.path.basename(FLAGS.DEMO_TYPE))
            # !! OpenCV can only write in .avi
            cv_writer = cv2.VideoWriter(
                writer_path + '.avi',
                # cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'),
                cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
                ori_fps,
                (FLAGS.input_size, FLAGS.input_size))
            # imageio_writer = imageio.get_writer(writer_path, fps=ori_fps)

            try:
                for it, im in enumerate(cam):
                    test_img_t = time.time()

                    test_img = cpm_utils.read_image(im, [], FLAGS.input_size,
                                                    'VIDEO')
                    test_img_resize = cv2.resize(
                        test_img, (FLAGS.input_size, FLAGS.input_size))
                    print('img read time %f' % (time.time() - test_img_t))

                    if FLAGS.color_channel == 'GRAY':
                        test_img_resize = mgray(test_img_resize, test_img)

                    test_img_input = test_img_resize / 256.0 - 0.5
                    test_img_input = np.expand_dims(test_img_input, axis=0)

                    # Inference
                    fps_t = time.time()
                    predict_heatmap, stage_heatmap_np = sess.run(
                        [
                            model.current_heatmap,
                            model.stage_heatmap,
                        ],
                        feed_dict={
                            'input_image:0': test_img_input,
                            'center_map:0': test_center_map
                        })

                    # Show visualized image
                    demo_img = visualize_result(test_img, FLAGS,
                                                stage_heatmap_np,
                                                kalman_filter_array)
                    cv2.imshow('demo_img', demo_img.astype(np.uint8))
                    if (cv2.waitKey(1) == ord('q')): break
                    print('fps: %.2f' % (1 / (time.time() - fps_t)))

                    cv_writer.write(demo_img.astype(np.uint8))
                    # imageio_writer.append_data(demo_img[:, :, 1])
            except KeyboardInterrupt:
                print('Stopped! {}/{} frames captured!'.format(
                    it, video_length))
            finally:
                cv_writer.release()
                # imageio_writer.close()
        else:
            while True:
                test_img_t = time.time()

                if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
                    test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [],
                                                    FLAGS.input_size, 'IMAGE')
                else:
                    test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
                                                    'WEBCAM')

                test_img_resize = cv2.resize(
                    test_img, (FLAGS.input_size, FLAGS.input_size))
                print('img read time %f' % (time.time() - test_img_t))

                if FLAGS.color_channel == 'GRAY':
                    test_img_resize = mgray(test_img_resize, test_img)

                test_img_input = test_img_resize / 256.0 - 0.5
                test_img_input = np.expand_dims(test_img_input, axis=0)

                if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
                    # Inference
                    fps_t = time.time()
                    predict_heatmap, stage_heatmap_np = sess.run(
                        [
                            model.current_heatmap,
                            model.stage_heatmap,
                        ],
                        feed_dict={
                            'input_image:0': test_img_input,
                            'center_map:0': test_center_map
                        })

                    # Show visualized image
                    demo_img = visualize_result(test_img, FLAGS,
                                                stage_heatmap_np,
                                                kalman_filter_array)
                    cv2.imshow('demo_img', demo_img.astype(np.uint8))
                    if cv2.waitKey(0) == ord('q'): break
                    print('fps: %.2f' % (1 / (time.time() - fps_t)))

                elif FLAGS.DEMO_TYPE == 'MULTI':

                    # Inference
                    fps_t = time.time()
                    predict_heatmap, stage_heatmap_np = sess.run(
                        [
                            model.current_heatmap,
                            model.stage_heatmap,
                        ],
                        feed_dict={
                            'input_image:0': test_img_input,
                            'center_map:0': test_center_map
                        })

                    # Show visualized image
                    demo_img = visualize_result(test_img, FLAGS,
                                                stage_heatmap_np,
                                                kalman_filter_array)
                    cv2.imshow('demo_img', demo_img.astype(np.uint8))
                    if cv2.waitKey(1) == ord('q'): break
                    print('fps: %.2f' % (1 / (time.time() - fps_t)))

                elif FLAGS.DEMO_TYPE == 'SINGLE':

                    # Inference
                    fps_t = time.time()
                    stage_heatmap_np = sess.run(
                        [model.stage_heatmap[5]],
                        feed_dict={
                            'input_image:0': test_img_input,
                            'center_map:0': test_center_map
                        })

                    # Show visualized image
                    demo_img = visualize_result(test_img, FLAGS,
                                                stage_heatmap_np,
                                                kalman_filter_array)
                    cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
                    if cv2.waitKey(1) == ord('q'): break
                    print('fps: %.2f' % (1 / (time.time() - fps_t)))

                elif FLAGS.DEMO_TYPE == 'HM':

                    # Inference
                    fps_t = time.time()
                    stage_heatmap_np = sess.run(
                        [model.stage_heatmap[FLAGS.stages - 1]],
                        feed_dict={
                            'input_image:0': test_img_input,
                            'center_map:0': test_center_map
                        })
                    print('fps: %.2f' % (1 / (time.time() - fps_t)))

                    # demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape(
                    #     (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
                    demo_stage_heatmap = stage_heatmap_np[-1][
                        0, :, :, 0:FLAGS.joints].reshape(
                            (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
                    demo_stage_heatmap = cv2.resize(
                        demo_stage_heatmap,
                        (FLAGS.input_size, FLAGS.input_size))

                    vertical_imgs = []
                    tmp_img = None
                    joint_coord_set = np.zeros((FLAGS.joints, 2))

                    for joint_num in range(FLAGS.joints):
                        # Concat until 4 img
                        if (joint_num % 4) == 0 and joint_num != 0:
                            vertical_imgs.append(tmp_img)
                            tmp_img = None

                        demo_stage_heatmap[:, :, joint_num] *= (
                            255 / np.max(demo_stage_heatmap[:, :, joint_num]))

                        # Plot color joints
                        if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
                            joint_coord = np.unravel_index(
                                np.argmax(demo_stage_heatmap[:, :, joint_num]),
                                (FLAGS.input_size, FLAGS.input_size))
                            joint_coord_set[joint_num, :] = joint_coord
                            color_code_num = (joint_num // 4)

                            if joint_num in [0, 4, 8, 12, 16]:
                                if PYTHON_VERSION == 3:
                                    joint_color = list(
                                        map(lambda x: x + 35 * (joint_num % 4),
                                            joint_color_code[color_code_num]))
                                else:
                                    joint_color = map(
                                        lambda x: x + 35 * (joint_num % 4),
                                        joint_color_code[color_code_num])

                                cv2.circle(test_img,
                                           center=(joint_coord[1],
                                                   joint_coord[0]),
                                           radius=3,
                                           color=joint_color,
                                           thickness=-1)
                            else:
                                if PYTHON_VERSION == 3:
                                    joint_color = list(
                                        map(lambda x: x + 35 * (joint_num % 4),
                                            joint_color_code[color_code_num]))
                                else:
                                    joint_color = map(
                                        lambda x: x + 35 * (joint_num % 4),
                                        joint_color_code[color_code_num])

                                cv2.circle(test_img,
                                           center=(joint_coord[1],
                                                   joint_coord[0]),
                                           radius=3,
                                           color=joint_color,
                                           thickness=-1)

                        # Put text
                        tmp = demo_stage_heatmap[:, :,
                                                 joint_num].astype(np.uint8)
                        tmp = cv2.putText(
                            tmp,
                            'Min:' +
                            str(np.min(demo_stage_heatmap[:, :, joint_num])),
                            org=(5, 20),
                            fontFace=cv2.FONT_HERSHEY_COMPLEX,
                            fontScale=0.3,
                            color=150)
                        tmp = cv2.putText(
                            tmp,
                            'Mean:' +
                            str(np.mean(demo_stage_heatmap[:, :, joint_num])),
                            org=(5, 30),
                            fontFace=cv2.FONT_HERSHEY_COMPLEX,
                            fontScale=0.3,
                            color=150)
                        tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
                            if tmp_img is not None else tmp

                    # Plot limbs
                    for limb_num in range(len(limbs)):
                        if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]
                                  ) > -2000 and np.min(
                                      demo_stage_heatmap[:, :,
                                                         limbs[limb_num][1]]
                                  ) > -2000:
                            x1 = joint_coord_set[limbs[limb_num][0], 0]
                            y1 = joint_coord_set[limbs[limb_num][0], 1]
                            x2 = joint_coord_set[limbs[limb_num][1], 0]
                            y2 = joint_coord_set[limbs[limb_num][1], 1]
                            length = ((x1 - x2)**2 + (y1 - y2)**2)**0.5
                            if length < 10000 and length > 5:
                                deg = math.degrees(math.atan2(
                                    x1 - x2, y1 - y2))
                                polygon = cv2.ellipse2Poly((int(
                                    (y1 + y2) / 2), int(
                                        (x1 + x2) / 2)), (int(length / 2), 3),
                                                           int(deg), 0, 360, 1)
                                color_code_num = limb_num // 4
                                if PYTHON_VERSION == 3:
                                    limb_color = list(
                                        map(lambda x: x + 35 * (limb_num % 4),
                                            joint_color_code[color_code_num]))
                                else:
                                    limb_color = map(
                                        lambda x: x + 35 * (limb_num % 4),
                                        joint_color_code[color_code_num])

                                cv2.fillConvexPoly(test_img,
                                                   polygon,
                                                   color=limb_color)

                    if tmp_img is not None:
                        tmp_img = np.lib.pad(
                            tmp_img,
                            ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]),
                             (0, 0)),
                            'constant',
                            constant_values=(0, 0))
                        vertical_imgs.append(tmp_img)

                    # Concat horizontally
                    output_img = None
                    for col in range(len(vertical_imgs)):
                        output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
                            vertical_imgs[col]

                    output_img = output_img.astype(np.uint8)
                    output_img = cv2.applyColorMap(output_img,
                                                   cv2.COLORMAP_JET)
                    test_img = cv2.resize(test_img, (300, 300),
                                          cv2.INTER_LANCZOS4)
                    cv2.imshow('hm', output_img)
                    cv2.moveWindow('hm', 2000, 200)
                    cv2.imshow('rgb', test_img)
                    cv2.moveWindow('rgb', 2000, 750)
                    if cv2.waitKey(1) == ord('q'): break
示例#34
0
def vis_frame(im_res, format='coco'):
    '''
    frame: frame image
    im_res: im_res of predictions
    format: coco or mpii

    return rendered image
    '''
    if format == 'coco':
        l_pair = [
            (0, 1), (0, 2), (1, 3), (2, 4),  # Head
            (5, 6), (5, 7), (7, 9), (6, 8), (8, 10),
            (17, 11), (17, 12),  # Body
            (11, 13), (12, 14), (13, 15), (14, 16)
        ]

        p_color = [(0, 255, 255), (0, 191, 255),(0, 255, 102),(0, 77, 255), (0, 255, 0), #Nose, LEye, REye, LEar, REar
                    (77,255,255), (77, 255, 204), (77,204,255), (191, 255, 77), (77,191,255), (191, 255, 77), #LShoulder, RShoulder, LElbow, RElbow, LWrist, RWrist
                    (204,77,255), (77,255,204), (191,77,255), (77,255,191), (127,77,255), (77,255,127), (0, 255, 255)] #LHip, RHip, LKnee, Rknee, LAnkle, RAnkle, Neck
        line_color = [(0, 215, 255), (0, 255, 204), (0, 134, 255), (0, 255, 50), 
                    (77,255,222), (77,196,255), (77,135,255), (191,255,77), (77,255,77), 
                    (77,222,255), (255,156,127), 
                    (0,127,255), (255,127,77), (0,77,255), (255,77,36)]
    elif format == 'mpii':
        l_pair = [
            (8, 9), (11, 12), (11, 10), (2, 1), (1, 0),
            (13, 14), (14, 15), (3, 4), (4, 5),
            (8, 7), (7, 6), (6, 2), (6, 3), (8, 12), (8, 13)
        ]
        p_color = [PURPLE, BLUE, BLUE, RED, RED, BLUE, BLUE, RED, RED, PURPLE, PURPLE, PURPLE, RED, RED, BLUE, BLUE]
        line_color = [PURPLE, BLUE, BLUE, RED, RED, BLUE, BLUE, RED, RED, PURPLE, PURPLE, RED, RED, BLUE, BLUE]
    else:
        raise NotImplementedError
        
    #print(im_res) #added by ashvini   
#     im_name = im_res['imgname'].split('/')[-1]
#     im_name = im_res[2]
    #print(frame.shape)
    img = np.zeros((360,640,3),np.uint8)  #image=frame #added by ashvini
    height,width = img.shape[:2]
    img = cv2.resize(img,(int(width/2), int(height/2)))
#     for human in im_res:
    part_line = {}
    kp_preds =  torch.Tensor(list(zip(im_res[3:20],im_res[20:37])))
    #added by ashvini start here
#     y = kp_preds[:,0]
#     x = kp_preds[:,1]
#     y_stand = 90.0
#     x_stand = 160.0
#     norm_y = max(y)-min(y)
#     norm_x = max(x)-min(x)
#     y_c = (max(y)+min(y))/2
#     x_c = (max(x)+min(x))/2
#     #print(norm_y, norm_x)
#     kp_preds[:,0] = (((kp_preds[:,0]-y_c)/norm_y)*y_stand)+y_c
#     kp_preds[:,1] = (((kp_preds[:,1]-x_c)/norm_x)*x_stand)+x_c
#     print(len(kp_preds))
    #print(kp_preds)
    #print("zero",kp_preds[:,0],type(kp_preds))
    #added by ashvini end here
    kp_scores =torch.Tensor(im_res[37:54]).unsqueeze(1)
#     print(kp_scores.shape)
    kp_preds = torch.cat((kp_preds, torch.unsqueeze((kp_preds[5,:]+kp_preds[6,:])/2,0)))
    kp_scores = torch.cat((kp_scores, torch.unsqueeze((kp_scores[5,:]+kp_scores[6,:])/2,0)))
    # Draw keypoints
    for n in range(kp_scores.shape[0]):
        if kp_scores[n] <= 0.05:
            continue
        cor_x, cor_y = int(kp_preds[n, 0]), int(kp_preds[n, 1])
        part_line[n] = (int(cor_x/2), int(cor_y/2))
        bg = img.copy()
        cv2.circle(bg, (int(cor_x/2), int(cor_y/2)), 2, p_color[n], -1)
        # Now create a mask of logo and create its inverse mask also
        transparency = max(0, min(1, kp_scores[n]))
        img = cv2.addWeighted(bg, transparency, img, 1-transparency, 0)
    # Draw limbs
    
    for i, (start_p, end_p) in enumerate(l_pair):
        if start_p in part_line and end_p in part_line:
            start_xy = part_line[start_p]
            end_xy = part_line[end_p]
            bg = img.copy()

            X = (start_xy[0], end_xy[0])
            Y = (start_xy[1], end_xy[1])
            mX = np.mean(X)
            mY = np.mean(Y)
            length = ((Y[0] - Y[1]) ** 2 + (X[0] - X[1]) ** 2) ** 0.5
            angle = math.degrees(math.atan2(Y[0] - Y[1], X[0] - X[1]))
            stickwidth = (kp_scores[start_p] + kp_scores[end_p]) + 1
            polygon = cv2.ellipse2Poly((int(mX),int(mY)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
            cv2.fillConvexPoly(bg, polygon, line_color[i])
            #cv2.line(bg, start_xy, end_xy, line_color[i], (2 * (kp_scores[start_p] + kp_scores[end_p])) + 1)
            transparency = max(0, min(1, 0.5*(kp_scores[start_p] + kp_scores[end_p])))
            img = cv2.addWeighted(bg, transparency, img, 1-transparency, 0)
    img = cv2.resize(img,(width,height),interpolation=cv2.INTER_CUBIC)
    return img
示例#35
0
def handle_one_run1(oriImg, model_, param_, model, mapIdx, limbSeq, colors):
    tic = time.time()
    # for visualize
    canvas = np.copy(oriImg)
    multiplier = [
        x * model_['boxsize'] / oriImg.shape[0] for x in param_['scale_search']
    ]
    imageToTest = Variable(T.transpose(
        T.transpose(T.unsqueeze(torch.from_numpy(oriImg).float(), 0), 2, 3), 1,
        2),
                           volatile=True).cuda()
    #print oriImg.shape
    scale = model_['boxsize'] / float(oriImg.shape[0])
    #print scale
    h = int(oriImg.shape[0] * scale)
    w = int(oriImg.shape[1] * scale)
    pad_h = 0 if (h % model_['stride']
                  == 0) else model_['stride'] - (h % model_['stride'])
    pad_w = 0 if (w % model_['stride']
                  == 0) else model_['stride'] - (w % model_['stride'])
    new_h = h + pad_h
    new_w = w + pad_w

    imageToTest = cv2.resize(oriImg, (0, 0),
                             fx=scale,
                             fy=scale,
                             interpolation=cv2.INTER_CUBIC)
    imageToTest_padded, pad = util.padRightDownCorner(imageToTest,
                                                      model_['stride'],
                                                      model_['padValue'])
    imageToTest_padded = np.transpose(
        np.float32(imageToTest_padded[:, :, :, np.newaxis]),
        (3, 2, 0, 1)) / 256 - 0.5

    feed = Variable(T.from_numpy(imageToTest_padded)).cuda()

    tic = time.time()
    out1, out2, out3, out4, out5, out5_1, out5_2, output1, output2, out6 = model(
        feed)

    heatmap = nn.UpsamplingBilinear2d(
        (oriImg.shape[0], oriImg.shape[1])).cuda()(output2)

    paf = nn.UpsamplingBilinear2d(
        (oriImg.shape[0], oriImg.shape[1])).cuda()(output1)
    #print '-------------------------------------'+str(time.time()-tic)

    #print heatmap.size()
    #print paf.size()
    #print type(heatmap)
    heatmap_avg = torch.zeros(
        (len(multiplier), 19, oriImg.shape[0], oriImg.shape[1])).cuda()
    paf_avg = torch.zeros(
        (len(multiplier), 38, oriImg.shape[0], oriImg.shape[1])).cuda()
    toc = time.time()

    #print 'time is %.5f'%(toc-tic)
    tic = time.time()
    for m in range(len(multiplier)):
        scale = multiplier[m]
        h = int(oriImg.shape[0] * scale)
        w = int(oriImg.shape[1] * scale)
        pad_h = 0 if (h % model_['stride']
                      == 0) else model_['stride'] - (h % model_['stride'])
        pad_w = 0 if (w % model_['stride']
                      == 0) else model_['stride'] - (w % model_['stride'])
        new_h = h + pad_h
        new_w = w + pad_w

        imageToTest = cv2.resize(oriImg, (0, 0),
                                 fx=scale,
                                 fy=scale,
                                 interpolation=cv2.INTER_CUBIC)
        imageToTest_padded, pad = util.padRightDownCorner(
            imageToTest, model_['stride'], model_['padValue'])
        imageToTest_padded = np.transpose(
            np.float32(imageToTest_padded[:, :, :, np.newaxis]),
            (3, 2, 0, 1)) / 256 - 0.5

        feed = Variable(T.from_numpy(imageToTest_padded)).cuda()
        out1, out2, out3, out4, out5, out5_1, out5_2, output1, output2, out6 = model(
            feed)
        #print output1.size()
        #print output2.size()
        heatmap = nn.UpsamplingBilinear2d(
            (oriImg.shape[0], oriImg.shape[1])).cuda()(output2)

        paf = nn.UpsamplingBilinear2d(
            (oriImg.shape[0], oriImg.shape[1])).cuda()(output1)

        heatmap_avg[m] = heatmap[0].data
        paf_avg[m] = paf[0].data

    toc = time.time()
    #print 'time is %.5f'%(toc-tic)
    tic = time.time()

    heatmap_avg = T.transpose(
        T.transpose(T.squeeze(T.mean(heatmap_avg, 0)), 0, 1), 1, 2).cuda()
    paf_avg = T.transpose(T.transpose(T.squeeze(T.mean(paf_avg, 0)), 0, 1), 1,
                          2).cuda()
    heatmap_avg = heatmap_avg.cpu().numpy()
    paf_avg = paf_avg.cpu().numpy()
    #toc =time.time()
    #print 'time is %.5f'%(toc-tic)
    #tic = time.time()

    all_peaks = []
    peak_counter = 0

    #maps =
    for part in range(18):
        map_ori = heatmap_avg[:, :, part]
        map = gaussian_filter(map_ori, sigma=3)

        map_left = np.zeros(map.shape)
        map_left[1:, :] = map[:-1, :]
        map_right = np.zeros(map.shape)
        map_right[:-1, :] = map[1:, :]
        map_up = np.zeros(map.shape)
        map_up[:, 1:] = map[:, :-1]
        map_down = np.zeros(map.shape)
        map_down[:, :-1] = map[:, 1:]

        peaks_binary = np.logical_and.reduce(
            (map >= map_left, map >= map_right, map >= map_up, map >= map_down,
             map > param_['thre1']))
        #    peaks_binary = T.eq(
        #    peaks = zip(T.nonzero(peaks_binary)[0],T.nonzero(peaks_binary)[0])

        peaks = zip(np.nonzero(peaks_binary)[1],
                    np.nonzero(peaks_binary)[0])  # note reverse

        peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
        id = range(peak_counter, peak_counter + len(peaks))
        peaks_with_score_and_id = [
            peaks_with_score[i] + (id[i], ) for i in range(len(id))
        ]

        all_peaks.append(peaks_with_score_and_id)
        peak_counter += len(peaks)

    connection_all = []
    special_k = []
    mid_num = 10

    for k in range(len(mapIdx)):
        score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]]
        candA = all_peaks[limbSeq[k][0] - 1]
        candB = all_peaks[limbSeq[k][1] - 1]
        nA = len(candA)
        nB = len(candB)
        indexA, indexB = limbSeq[k]
        if (nA != 0 and nB != 0):
            connection_candidate = []
            for i in range(nA):
                for j in range(nB):
                    vec = np.subtract(candB[j][:2], candA[i][:2])
                    norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
                    vec = np.divide(vec, norm)

                    startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
                                   np.linspace(candA[i][1], candB[j][1], num=mid_num))

                    vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
                                      for I in range(len(startend))])
                    vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
                                      for I in range(len(startend))])

                    score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(
                        vec_y, vec[1])
                    if norm == 0:
                        score_with_dist_prior = sum(score_midpts) / len(
                            score_midpts)
                    else:
                        score_with_dist_prior = sum(score_midpts) / len(
                            score_midpts) + min(
                                0.5 * oriImg.shape[0] / norm - 1, 0)
                    criterion1 = len(
                        np.nonzero(score_midpts > param_['thre2'])
                        [0]) > 0.8 * len(score_midpts)
                    criterion2 = score_with_dist_prior > 0
                    if criterion1 and criterion2:
                        connection_candidate.append([
                            i, j, score_with_dist_prior,
                            score_with_dist_prior + candA[i][2] + candB[j][2]
                        ])

            connection_candidate = sorted(connection_candidate,
                                          key=lambda x: x[2],
                                          reverse=True)
            connection = np.zeros((0, 5))
            for c in range(len(connection_candidate)):
                i, j, s = connection_candidate[c][0:3]
                if (i not in connection[:, 3] and j not in connection[:, 4]):
                    connection = np.vstack(
                        [connection, [candA[i][3], candB[j][3], s, i, j]])
                    if (len(connection) >= min(nA, nB)):
                        break

            connection_all.append(connection)
        else:
            special_k.append(k)
            connection_all.append([])

    # last number in each row is the total parts number of that person
    # the second last number in each row is the score of the overall configuration
    subset = -1 * np.ones((0, 20))
    candidate = np.array([item for sublist in all_peaks for item in sublist])

    #print 'candidate is ', candidate
    for k in range(len(mapIdx)):

        if k not in special_k:
            partAs = connection_all[k][:, 0]
            partBs = connection_all[k][:, 1]
            indexA, indexB = np.array(limbSeq[k]) - 1

            for i in range(len(connection_all[k])):  #= 1:size(temp,1)
                found = 0
                subset_idx = [-1, -1]
                for j in range(len(subset)):  #1:size(subset,1):
                    if subset[j][indexA] == partAs[i] or subset[j][
                            indexB] == partBs[i]:
                        subset_idx[found] = j
                        found += 1

                if found == 1:
                    j = subset_idx[0]
                    if (subset[j][indexB] != partBs[i]):
                        subset[j][indexB] = partBs[i]
                        subset[j][-1] += 1
                        subset[j][-2] += candidate[partBs[i].astype(int),
                                                   2] + connection_all[k][i][2]
                elif found == 2:  # if found 2 and disjoint, merge them
                    j1, j2 = subset_idx
                    #print "found = 2"
                    membership = ((subset[j1] >= 0).astype(int) +
                                  (subset[j2] >= 0).astype(int))[:-2]
                    if len(np.nonzero(membership == 2)[0]) == 0:  #merge
                        subset[j1][:-2] += (subset[j2][:-2] + 1)
                        subset[j1][-2:] += subset[j2][-2:]
                        subset[j1][-2] += connection_all[k][i][2]
                        subset = np.delete(subset, j2, 0)
                    else:  # as like found == 1
                        subset[j1][indexB] = partBs[i]
                        subset[j1][-1] += 1
                        subset[j1][-2] += candidate[
                            partBs[i].astype(int), 2] + connection_all[k][i][2]

                # if find no partA in the subset, create a new subset
                elif not found and k < 17:
                    row = -1 * np.ones(20)
                    row[indexA] = partAs[i]
                    row[indexB] = partBs[i]
                    row[-1] = 2
                    row[-2] = sum(
                        candidate[connection_all[k][i, :2].astype(int),
                                  2]) + connection_all[k][i][2]
                    subset = np.vstack([subset, row])

    # delete some rows of subset which has few parts occur
    deleteIdx = []
    for i in range(len(subset)):
        if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
            deleteIdx.append(i)
    #temp = np.delete(subset, deleteIdx, axis=0)

#    canvas = cv2.imread(test_image) # B,G,R order
    for i in range(18):
        for j in range(len(all_peaks[i])):
            cv2.circle(canvas,
                       all_peaks[i][j][0:2],
                       4,
                       colors[i],
                       thickness=-1)

    stickwidth = 4
    output_coordinates = np.zeros((len(subset), 18, 2))

    for i in range(17):
        for n in range(len(subset)):
            index = subset[n][np.array(limbSeq[i]) - 1]

            if -1 in index:
                continue
            cur_canvas = canvas.copy()
            Y = candidate[index.astype(int), 0]

            X = candidate[index.astype(int), 1]

            mX = np.mean(X)
            mY = np.mean(Y)
            length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5
            angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
            polygon = cv2.ellipse2Poly(
                (int(mY), int(mX)), (int(length / 2), stickwidth), int(angle),
                0, 360, 1)
            cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
            canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
            output_coordinates[n, i, 0] = mX
            output_coordinates[n, i, 1] = mY

    return canvas, output_coordinates, out1, out2, out3, out4, out5, out6
示例#36
0
def generate_jointsmap(uv_coord, depth, width, height, channel=3):
    canvas = np.zeros((height, width))
    bones = [((17, 18), [130] * channel), ((18, 19), [140] * channel),
             ((19, 20), [150] * channel), ((1, 2), [10] * channel),
             ((2, 3), [20] * channel), ((3, 4), [30] * channel),
             ((5, 6), [40] * channel), ((6, 7), [50] * channel),
             ((7, 8), [60] * channel), ((9, 10), [70] * channel),
             ((10, 11), [80] * channel), ((11, 12), [90] * channel),
             ((13, 14), [100] * channel), ((14, 15), [110] * channel),
             ((15, 16), [120] * channel), ((0, 17), []), ((0, 1), []),
             ((0, 5), []), ((0, 9), []), ((0, 13), [])]
    palm = []
    for connection, _ in [
        ((0, 17), []),
        ((17, 1), []),
        ((1, 5), []),
        ((5, 9), []),
        ((9, 13), []),
        ((13, 0), []),
    ]:
        coord1 = uv_coord[connection[0]]
        palm.append([int(coord1[0]), int(coord1[1])])
    # palm.append([int((coord1[0]-.5)* W_scale+ W_offset ), int(-(coord1[1]- .5)* H_scale+ H_offset)])
    # print(palm)
    palm_colors = [
        depth[0], depth[17], depth[1], depth[5], depth[9], depth[13]
    ]
    palm_colors = list(filter(lambda x: x >= 0, palm_colors))
    # if len(palm_colors) == 6:
    #     binary_mask = np.zeros((height, width))
    #     cv2.fillConvexPoly(binary_mask, np.array([palm], dtype=np.int32), 1)
    #
    #     avg_color_upper = np.average(palm_colors[2::])
    #     avg_color_lower = np.average(palm_colors[::2])
    #
    #     Xs, Ys = np.array(palm).T
    #     Xmax, Xmin = np.max(Xs), np.min(Xs)
    #     Ymax, Ymin = np.max(Ys), np.min(Ys)
    #
    #     orientation = None
    #     s_c = None # start_color
    #     d_c = None # end_color
    #     wrist = np.array([palm[0], palm[1]]).T
    #     if Xmax in wrist[0]:
    #         orientation = "leftRight"
    #         s_c = avg_color_upper
    #         d_c = avg_color_lower
    #     elif Xmin in wrist[0]:
    #         orientation = "RightLeft"
    #         s_c = avg_color_lower
    #         d_c = avg_color_upper
    #     elif Ymax in wrist[1]:
    #         orientation = "TopDown"
    #         s_c = avg_color_upper
    #         d_c = avg_color_lower
    #     else:
    #         orientation = "BottomUp"
    #         s_c = avg_color_lower
    #         d_c = avg_color_upper
    #
    #     n_step = Xmax - Xmin if 'left' in orientation.lower() else Ymax - Ymin
    #
    #     gradient_offset = np.abs(avg_color_lower - avg_color_upper) / n_step
    #
    #     def add(x, y):
    #         return x+y
    #     def minus(x,y):
    #         return x-y
    #
    #     color_operation = minus if s_c > d_c else add
    #
    #     for i in range(int(n_step)):
    #         s_c = color_operation(s_c, i * gradient_offset)
    #         if 'left' in orientation.lower():
    #             canvas[:, Xmin + i] = s_c
    #         else:
    #             canvas[Ymin +i, :] = s_c
    #
    #     canvas = np.multiply(canvas, binary_mask)
    # else:
    if len(palm_colors):
        pass
        # cv2.fillPoly(canvas, np.array([palm], dtype=np.int32), np.average(palm_colors))

    for connection, color in bones:
        temp_canvas = np.zeros(canvas.shape)
        coord1 = uv_coord[connection[0]]
        coord2 = uv_coord[connection[1]]
        coords = np.stack([coord1, coord2])
        colors = [depth[connection[0]], depth[connection[1]]]
        if -1 in colors or len(colors) == 0:
            continue
        else:
            color = np.average(colors)
        # 0.5, 0.5 is the center
        x = coords[:, 0]
        y = coords[:, 1]
        mX = x.mean()
        mY = y.mean()
        length = ((x[0] - x[1])**2 + (y[0] - y[1])**2)**0.5
        angle = np.math.degrees(np.math.atan2(y[0] - y[1], x[0] - x[1]))
        radius = 2
        polygon = cv2.ellipse2Poly(
            (int(mX), int(mY)), (int(length / 3), radius), int(angle), 0, 360,
            1)
        cv2.fillConvexPoly(temp_canvas, polygon, color)
        canvas = np.maximum(canvas, temp_canvas)

    return canvas
示例#37
0
def segment_region(img, out_path, cnt, dist_per_pixel):
    """
    Find regions containing cells in the image

    :param img: Image object
    :param dist_per_pixel: How big wide each pixel is
    :return: List of regions
    """
    area_per_pixel = dist_per_pixel * dist_per_pixel
    img_h, img_w = img.shape[:2]
    green = img[:, :, 1]
    blurred = cv2.GaussianBlur(green, (BR_DEF, BR_DEF), 0)
    # Apply a threshold to remove background noise
    _, remaining = cv2.threshold(blurred, NT_MIN, 0, cv2.THRESH_TOZERO)

    contours = []
    noise_thresh = NT_MIN
    while np.any(remaining):
        # DEBUGGING
        # cv2.imwrite('img/remaining.png', remaining)
        # END
        _, remaining = cv2.threshold(remaining, noise_thresh, 0,
                                     cv2.THRESH_TOZERO)
        _, prepared = cv2.threshold(remaining, noise_thresh, 255,
                                    cv2.THRESH_BINARY)
        _, remaining_contours, _ = cv2.findContours(prepared,
                                                    cv2.RETR_EXTERNAL,
                                                    cv2.CHAIN_APPROX_SIMPLE)
        too_small = []
        for contour in remaining_contours:
            # # DEBUGGING CODE ONLY - VERY SLOW
            # cpy = img.copy()
            # cv2.drawContours(cpy, [contour], -1, WHITE)
            # cv2.imwrite('img/cntr.png', cpy)

            # Determine smallest rectangle that surrounds contour
            x, y, w, h = cv2.boundingRect(contour)

            # Ignore cells touching edges
            if x == 0 or y == 0 or x + w == img_w or y + h == img_h:
                cv2.drawContours(remaining, [contour], 0, 0)
                cv2.fillPoly(remaining, [contour], 0)
                # when dealing with images containing only one cell,
                # the cell might touch the edge
                continue

            area = w * h * area_per_pixel
            if int(1.4 * MAX_AREA) <= area:
                continue
            elif int(0.1 * MIN_AREA) <= area:
                cv2.drawContours(remaining, [contour], 0, 0)
                cv2.fillPoly(remaining, [contour], 0)
                if area <= int(0.6 * MIN_AREA):
                    too_small.append(contour)
                else:
                    contours.append(contour)
            else:
                cv2.drawContours(remaining, [contour], 0, 0)
                cv2.fillPoly(remaining, [contour], 0)

        contours += merge_contours(too_small, area_per_pixel)
        noise_thresh += 6

    # Output the contours found
    for contour in contours:

        x, y, w, h = cv2.boundingRect(contour)
        crypt = img[y:y + h, x:x + w].copy()

        red_part = crypt[:, :, 2]
        grn_chnl = crypt[:, :, 1]
        centre_y = h // 2
        centre_x = w // 2
        grn_cntr = grn_chnl[centre_y - centre_y // 2:centre_y + centre_y // 2,
                            centre_x - centre_x // 2:centre_x + centre_x // 2]

        _, red_nse_rmvd = cv2.threshold(red_part, 40, 255, cv2.THRESH_TOZERO)
        _, grn_nse_rmvd = cv2.threshold(grn_cntr, 20, 255, cv2.THRESH_TOZERO)
        # Red must account for > 2% of area to be considered significant

        if cv2.countNonZero(red_nse_rmvd) > (w * h) / 20 and \
                cv2.countNonZero(grn_nse_rmvd) > (w * h) / 16:

            SCORING_AREA = 0.2
            x_to_score = max(0, int(x - SCORING_AREA * w))
            y_to_score = max(0, int(y - SCORING_AREA * h))
            w_to_score = min(img_w - x_to_score,
                             int(w * (1 + 2 * SCORING_AREA)))
            h_to_score = min(img_h - y_to_score,
                             int(h * (1 + 2 * SCORING_AREA)))

            # Extract area we want our scorer to see
            cell_to_score = img[y_to_score:y_to_score + h_to_score,
                                x_to_score:x_to_score + w_to_score].copy()
            scoring_mask = np.zeros_like(cell_to_score)
            half_w = w_to_score // 2
            half_h = h_to_score // 2
            ellipse = cv2.ellipse2Poly((half_w, half_h), (half_w, half_h), 0,
                                       0, 360, 10)
            cv2.fillPoly(scoring_mask, [ellipse], WHITE)
            cell_to_score = cv2.bitwise_and(cell_to_score, scoring_mask)
            cv2.imwrite(out_path + str(cnt) + '.png', cell_to_score)

            cnt += 1
    return cnt
示例#38
0
    def recent_events(self, events):
        frame = events.get('frame')
        if not frame:
            return
        for eye_index in self.showeyes:
            requested_eye_frame_idx = self.eye_world_frame_map[eye_index][frame.index]

            #1. do we need a new frame?
            if requested_eye_frame_idx != self.eye_frames[eye_index].index:
                # do we need to seek?
                if requested_eye_frame_idx == self.eye_cap[eye_index].get_frame_index()+1:
                    # if we just need to seek by one frame, its faster to just read one and and throw it away.
                    _ = self.eye_cap[eye_index].get_frame()
                if requested_eye_frame_idx != self.eye_cap[eye_index].get_frame_index():
                    # only now do I need to seek
                    self.eye_cap[eye_index].seek_to_frame(requested_eye_frame_idx)
                # reading the new eye frame frame
                try:
                    self.eye_frames[eye_index] = self.eye_cap[eye_index].get_frame()
                except EndofVideoFileError:
                    logger.warning("Reached the end of the eye video for eye video {}.".format(eye_index))
            else:
                #our old frame is still valid because we are doing upsampling
                pass

            #2. dragging image
            if self.drag_offset[eye_index] is not None:
                pos = glfwGetCursorPos(glfwGetCurrentContext())
                pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext()))
                pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
                self.pos[eye_index][0] = pos[0]+self.drag_offset[eye_index][0]
                self.pos[eye_index][1] = pos[1]+self.drag_offset[eye_index][1]
            else:
                self.video_size = [round(self.eye_frames[eye_index].width*self.eye_scale_factor), round(self.eye_frames[eye_index].height*self.eye_scale_factor)]

            #3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
            self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen
            self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0))

            #4. flipping images, converting to greyscale
            eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling
            eyeimage = cv2.resize(eye_gray,(0,0),fx=self.eye_scale_factor, fy=self.eye_scale_factor)
            if self.mirror[str(eye_index)]:
                eyeimage = np.fliplr(eyeimage)
            if self.flip[str(eye_index)]:
                eyeimage = np.flipud(eyeimage)

            eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR)

            if self.show_ellipses and events['pupil_positions']:
                for pd in events['pupil_positions']:
                    if pd['id'] == eye_index and pd['timestamp'] == self.eye_frames[eye_index].timestamp:
                        el = pd['ellipse']
                        conf = int(pd.get('model_confidence', pd.get('confidence', 0.1)) * 255)
                        center = list(map(lambda val: int(self.eye_scale_factor*val), el['center']))
                        el['axes'] = tuple(map(lambda val: int(self.eye_scale_factor*val/2), el['axes']))
                        el['angle'] = int(el['angle'])
                        el_points = cv2.ellipse2Poly(tuple(center), el['axes'], el['angle'], 0, 360, 1)

                        if self.mirror[str(eye_index)]:
                            el_points = [(self.video_size[0] - x, y) for x, y in el_points]
                            center[0] = self.video_size[0] - center[0]
                        if self.flip[str(eye_index)]:
                            el_points = [(x, self.video_size[1] - y) for x, y in el_points]
                            center[1] = self.video_size[1] - center[1]

                        cv2.polylines(eyeimage, [np.asarray(el_points)], True, (0, 0, 255, conf), thickness=math.ceil(2*self.eye_scale_factor))
                        cv2.circle(eyeimage, tuple(center), int(5*self.eye_scale_factor), (0, 0, 255, conf), thickness=-1)

            # 5. finally overlay the image
            x, y = int(self.pos[eye_index][0]), int(self.pos[eye_index][1])
            transparent_image_overlay((x, y), eyeimage, frame.img, self.alpha)
示例#39
0
def plot_poses(img,
               skeletons,
               config=plt_config('coco'),
               save_path=None,
               dataset_name='coco'):
    # std = torch.tensor([0.229, 0.224, 0.225]).unsqueeze(-1).unsqueeze(-1)
    # mean = torch.tensor([0.485, 0.456, 0.406]).unsqueeze(-1).unsqueeze(-1)
    # # input = input.detach().cpu()
    # #if dataset_name == 'coco':
    # img = img * std + mean
    # img = img.permute(1, 2, 0).detach().numpy()  # [H,W,3]
    # colors = [[255, 0, 0],
    #           [155, 85, 0],
    #           [255, 170, 0],
    #           [255, 55, 0],
    #           [17, 25, 10],
    #           [85, 55, 0],
    #           [0, 255, 0],
    #           [0, 255, 5],
    #           [0, 255, 170],
    #           [0, 255, 25],
    #           [0, 17, 255],
    #           [0, 185, 255],
    #           [0, 0, 255],
    #           [125, 0, 255],
    #           [170, 0, 25],
    #           [25, 0, 255],
    #           [255, 0, 170],
    #           [255, 0, 85],
    #           [0, 170, 255],
    #           [0, 85, 255],
    #           [0, 0, 255],
    #           [85, 0, 255],
    #           [70, 0, 255],
    #           [25, 0, 255],
    #           [255, 0, 70],
    #           [55, 10, 5]]

    cmap = matplotlib.cm.get_cmap('hsv')
    canvas = img.copy()
    n_kpt = config.n_kpt
    for i in range(n_kpt):
        rgba = np.array(cmap(1 - i / n_kpt - 1. / n_kpt * 2))
        rgba[0:3] *= 255
        for j in range(len(skeletons)):
            if len(skeletons[j][i]) > 2 and skeletons[j][i, 2] > 0:
                cv2.circle(canvas,
                           tuple(skeletons[j][i, 0:2].astype('int32')),
                           3, (255, 255, 255),
                           thickness=-1)

    # to_plot = cv2.addWeighted(img, 0.3, canvas, 0.7, 0)
    stickwidth = 2

    for i in range(len(config.EDGES)):
        for j in range(len(skeletons)):
            edge = config.EDGES[i][0]
            color = config.EDGES[i][1]
            if len(skeletons[j][edge[0]]) > 2:
                if skeletons[j][edge[0], 2] == 0 or skeletons[j][edge[1],
                                                                 2] == 0:
                    continue

            cur_canvas = canvas.copy()
            X = [skeletons[j][edge[0], 1], skeletons[j][edge[1], 1]]
            Y = [skeletons[j][edge[0], 0], skeletons[j][edge[1], 0]]
            mX = np.mean(X)
            mY = np.mean(Y)
            length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5
            angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
            polygon = cv2.ellipse2Poly(
                (int(mY), int(mX)), (int(length / 2), stickwidth), int(angle),
                0, 360, 1)
            cv2.fillConvexPoly(cur_canvas, polygon, color)
            canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)

    return canvas
示例#40
0
文件: eye.py 项目: NoahMilam/pupil
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src):
    """
    Creates a window, gl context.
    Grabs images from a capture.
    Streams Pupil coordinates into g_pool.pupil_queue
    """
    is_alive = Is_Alive_Manager(is_alive_flag)
    with is_alive:
        import logging
        # Set up root logger for this process before doing imports of logged modules.
        logger = logging.getLogger()
        logger.setLevel(logging.INFO)
        # remove inherited handlers
        logger.handlers = []
        # create file handler which logs even debug messages
        fh = logging.FileHandler(os.path.join(user_dir,'eye%s.log'%eye_id),mode='w')
        # fh.setLevel(logging.DEBUG)
        # create console handler with a higher log level
        ch = logging.StreamHandler()
        ch.setLevel(logger.level+10)
        # create formatter and add it to the handlers
        formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
        ch.setFormatter(formatter)
        # add the handlers to the logger
        logger.addHandler(fh)
        logger.addHandler(ch)
        #silence noisy modules
        logging.getLogger("OpenGL").setLevel(logging.ERROR)
        # create logger for the context of this function
        logger = logging.getLogger(__name__)


        # We deferr the imports becasue of multiprocessing.
        # Otherwise the world process each process also loads the other imports.

        #general imports
        import numpy as np
        import cv2

        #display
        import glfw
        from pyglui import ui,graph,cygl
        from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
        import OpenGL.GL as gl
        from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based
        from ui_roi import UIRoi
        #monitoring
        import psutil
        import math


        # helpers/utils
        from file_methods import Persistent_Dict
        from version_utils import VersionFormat
        from methods import normalize, denormalize, Roi, timer
        from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
        from av_writer import JPEG_Writer,AV_Writer

        # Pupil detectors
        from pupil_detectors import Detector_2D, Detector_3D
        pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}



        #UI Platform tweaks
        if platform.system() == 'Linux':
            scroll_factor = 10.0
            window_position_default = (600,300*eye_id)
        elif platform.system() == 'Windows':
            scroll_factor = 1.0
            window_position_default = (600,31+300*eye_id)
        else:
            scroll_factor = 1.0
            window_position_default = (600,300*eye_id)


        #g_pool holds variables for this process
        g_pool = Global_Container()

        # make some constants avaiable
        g_pool.user_dir = user_dir
        g_pool.version = version
        g_pool.app = 'capture'
        g_pool.pupil_queue = pupil_queue
        g_pool.timebase = timebase


        # Callback functions
        def on_resize(window,w, h):
            if not g_pool.iconified:
                active_window = glfw.glfwGetCurrentContext()
                glfw.glfwMakeContextCurrent(window)
                g_pool.gui.update_window(w,h)
                graph.adjust_size(w,h)
                adjust_gl_view(w,h)
                glfw.glfwMakeContextCurrent(active_window)

        def on_key(window, key, scancode, action, mods):
            g_pool.gui.update_key(key,scancode,action,mods)

        def on_char(window,char):
            g_pool.gui.update_char(char)

        def on_iconify(window,iconified):
            g_pool.iconified = iconified

        def on_button(window,button, action, mods):
            if g_pool.display_mode == 'roi':
                if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
                    g_pool.u_r.active_edit_pt = False
                    return # if the roi interacts we dont what the gui to interact as well
                elif action == glfw.GLFW_PRESS:
                    pos = glfw.glfwGetCursorPos(window)
                    pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
                    if g_pool.flip:
                        pos = 1-pos[0],1-pos[1]
                    pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
                    if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
                        return # if the roi interacts we dont what the gui to interact as well

            g_pool.gui.update_button(button,action,mods)



        def on_pos(window,x, y):
            hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
            g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)

            if g_pool.u_r.active_edit_pt:
                pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
                if g_pool.flip:
                    pos = 1-pos[0],1-pos[1]
                pos = denormalize(pos,(frame.width,frame.height) )
                g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)

        def on_scroll(window,x,y):
            g_pool.gui.update_scroll(x,y*scroll_factor)


        # load session persistent settings
        session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
        if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
            logger.info("Session setting are from older version of this app. I will not use those.")
            session_settings.clear()
        # Initialize capture
        cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
        default_settings = {'frame_size':(640,480),'frame_rate':60}
        previous_settings = session_settings.get('capture_settings',None)
        if previous_settings and previous_settings['name'] == cap.name:
            cap.settings = previous_settings
        else:
            cap.settings = default_settings


        # Test capture
        try:
            frame = cap.get_frame()
        except CameraCaptureError:
            logger.error("Could not retrieve image from capture")
            cap.close()
            return

        #signal world that we are ready to go
        # pipe_to_world.send('eye%s process ready'%eye_id)

        # any object we attach to the g_pool object *from now on* will only be visible to this process!
        # vars should be declared here to make them visible to the code reader.
        g_pool.iconified = False
        g_pool.capture = cap
        g_pool.flip = session_settings.get('flip',False)
        g_pool.display_mode = session_settings.get('display_mode','camera_image')
        g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
                                    'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
                                    'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}


        g_pool.u_r = UIRoi(frame.img.shape)
        g_pool.u_r.set(session_settings.get('roi',g_pool.u_r.get()))


        def on_frame_size_change(new_size):
            g_pool.u_r = UIRoi((new_size[1],new_size[0]))

        cap.on_frame_size_change = on_frame_size_change

        writer = None

        pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
        last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
        g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)

        # UI callback functions
        def set_scale(new_scale):
            g_pool.gui.scale = new_scale
            g_pool.gui.collect_menus()


        def set_display_mode_info(val):
            g_pool.display_mode = val
            g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]


        def set_detector(new_detector):
            g_pool.pupil_detector.cleanup()
            g_pool.pupil_detector = new_detector(g_pool)
            g_pool.pupil_detector.init_gui(g_pool.sidebar)


        # Initialize glfw
        glfw.glfwInit()
        title = "eye %s"%eye_id
        width,height = session_settings.get('window_size',(frame.width, frame.height))
        main_window = glfw.glfwCreateWindow(width,height, title, None, None)
        window_pos = session_settings.get('window_position',window_position_default)
        glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
        glfw.glfwMakeContextCurrent(main_window)
        cygl.utils.init()

        # gl_state settings
        basic_gl_setup()
        g_pool.image_tex = Named_Texture()
        g_pool.image_tex.update_from_frame(frame)
        glfw.glfwSwapInterval(0)

        sphere  = Sphere(20)

        #setup GUI
        g_pool.gui = ui.UI()
        g_pool.gui.scale = session_settings.get('gui_scale',1)
        g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
        general_settings = ui.Growing_Menu('General')
        general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
        general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
        general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
        general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
        g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
        general_settings.append(g_pool.display_mode_info)
        g_pool.sidebar.append(general_settings)
        g_pool.gui.append(g_pool.sidebar)
        detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
        general_settings.append(detector_selector)

        # let detector add its GUI
        g_pool.pupil_detector.init_gui(g_pool.sidebar)
        # let the camera add its GUI
        g_pool.capture.init_gui(g_pool.sidebar)


        # Register callbacks main_window
        glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
        glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
        glfw.glfwSetKeyCallback(main_window,on_key)
        glfw.glfwSetCharCallback(main_window,on_char)
        glfw.glfwSetMouseButtonCallback(main_window,on_button)
        glfw.glfwSetCursorPosCallback(main_window,on_pos)
        glfw.glfwSetScrollCallback(main_window,on_scroll)

        #set the last saved window size
        on_resize(main_window, *glfw.glfwGetWindowSize(main_window))


        # load last gui configuration
        g_pool.gui.configuration = session_settings.get('ui_config',{})


        #set up performance graphs
        pid = os.getpid()
        ps = psutil.Process(pid)
        ts = frame.timestamp

        cpu_graph = graph.Bar_Graph()
        cpu_graph.pos = (20,130)
        cpu_graph.update_fn = ps.cpu_percent
        cpu_graph.update_rate = 5
        cpu_graph.label = 'CPU %0.1f'

        fps_graph = graph.Bar_Graph()
        fps_graph.pos = (140,130)
        fps_graph.update_rate = 5
        fps_graph.label = "%0.0f FPS"


        #create a timer to control window update frequency
        window_update_timer = timer(1/60.)
        def window_should_update():
            return next(window_update_timer)


        # Event loop
        while not glfw.glfwWindowShouldClose(main_window):

            if pipe_to_world.poll():
                cmd = pipe_to_world.recv()
                if cmd == 'Exit':
                    break
                elif cmd == "Ping":
                    pipe_to_world.send("Pong")
                    command = None
                else:
                    command,payload = cmd
                if command == 'Set_Detection_Mapping_Mode':
                    if payload == '3d':
                        if not isinstance(g_pool.pupil_detector,Detector_3D):
                            set_detector(Detector_3D)
                        detector_selector.read_only  = True
                    else:
                        set_detector(Detector_2D)
                        detector_selector.read_only = False

            else:
                command = None



            # Get an image from the grabber
            try:
                frame = cap.get_frame()
            except CameraCaptureError:
                logger.error("Capture from Camera Failed. Stopping.")
                break
            except EndofVideoFileError:
                logger.warning("Video File is done. Stopping")
                cap.seek_to_frame(0)
                frame = cap.get_frame()


            #update performace graphs
            t = frame.timestamp
            dt,ts = t-ts,t
            try:
                fps_graph.add(1./dt)
            except ZeroDivisionError:
                pass
            cpu_graph.update()


            ###  RECORDING of Eye Video (on demand) ###
            # Setup variables and lists for recording
            if 'Rec_Start' == command:
                record_path,raw_mode = payload
                logger.info("Will save eye video to: %s"%record_path)
                timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
                if raw_mode and frame.jpeg_buffer:
                    video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                    writer = JPEG_Writer(video_path,cap.frame_rate)
                else:
                    video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
                    writer = AV_Writer(video_path,cap.frame_rate)
                timestamps = []
            elif 'Rec_Stop' == command:
                logger.info("Done recording.")
                writer.release()
                writer = None
                np.save(timestamps_path,np.asarray(timestamps))
                del timestamps

            if writer:
                writer.write_video_frame(frame)
                timestamps.append(frame.timestamp)


            # pupil ellipse detection
            result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
            result['id'] = eye_id
            # stream the result
            g_pool.pupil_queue.put(result)

            # GL drawing
            if window_should_update():
                if not g_pool.iconified:
                    glfw.glfwMakeContextCurrent(main_window)
                    clear_gl_screen()

                    # switch to work in normalized coordinate space
                    if g_pool.display_mode == 'algorithm':
                        g_pool.image_tex.update_from_ndarray(frame.img)
                    elif g_pool.display_mode in ('camera_image','roi'):
                        g_pool.image_tex.update_from_ndarray(frame.gray)
                    else:
                        pass

                    make_coord_system_norm_based(g_pool.flip)
                    g_pool.image_tex.draw()

                    window_size =  glfw.glfwGetWindowSize(main_window)
                    make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)

                    if result['method'] == '3d c++':

                        eye_ball = result['projected_sphere']
                        try:
                            pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
                                                (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
                                                int(eye_ball['angle']),0,360,8)
                        except ValueError as e:
                            pass
                        else:
                            draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )

                    if result['confidence'] >0:
                        if result.has_key('ellipse'):
                            pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
                                            (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
                                            int(result['ellipse']['angle']),0,360,15)
                            confidence = result['confidence'] * 0.7 #scale it a little
                            draw_polyline(pts,1,RGBA(1.,0,0,confidence))
                            draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)

                    # render graphs
                    graph.push_view()
                    fps_graph.draw()
                    cpu_graph.draw()
                    graph.pop_view()

                    # render GUI
                    g_pool.gui.update()

                    #render the ROI
                    if g_pool.display_mode == 'roi':
                        g_pool.u_r.draw(g_pool.gui.scale)

                    #update screen
                    glfw.glfwSwapBuffers(main_window)
                glfw.glfwPollEvents()
                g_pool.pupil_detector.visualize() #detector decides if we visualize or not


        # END while running

        # in case eye recording was still runnnig: Save&close
        if writer:
            logger.info("Done recording eye.")
            writer = None
            np.save(timestamps_path,np.asarray(timestamps))

        glfw.glfwRestoreWindow(main_window) #need to do this for windows os
        # save session persistent settings
        session_settings['gui_scale'] = g_pool.gui.scale
        session_settings['roi'] = g_pool.u_r.get()
        session_settings['flip'] = g_pool.flip
        session_settings['display_mode'] = g_pool.display_mode
        session_settings['ui_config'] = g_pool.gui.configuration
        session_settings['capture_settings'] = g_pool.capture.settings
        session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
        session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
        session_settings['version'] = g_pool.version
        session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
        session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
        session_settings.close()

        g_pool.pupil_detector.cleanup()
        g_pool.gui.terminate()
        glfw.glfwDestroyWindow(main_window)
        glfw.glfwTerminate()
        cap.close()


        logger.debug("Process done")
示例#41
0
    def predict(self, test_image, idx):
        #test_image = '../sample_image/ski.jpg'
        #test_image = '../sample_image/upper.jpg'
        #test_image = '../sample_image/upper2.jpg'
        oriImg = cv.imread(test_image)  # B,G,R order

        param, model = config_reader()
        multiplier = [
            x * model['boxsize'] / oriImg.shape[0]
            for x in param['scale_search']
        ]

        heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 15))
        paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 26))
        # first figure shows padded images

        for m in range(len(multiplier)):
            scale = multiplier[m]
            imageToTest = cv.resize(oriImg, (0, 0),
                                    fx=scale,
                                    fy=scale,
                                    interpolation=cv.INTER_CUBIC)
            imageToTest_padded, pad = util.padRightDownCorner(
                imageToTest, model['stride'], model['padValue'])
            print imageToTest_padded.shape

            self.net.blobs['data'].reshape(*(1, 3, imageToTest_padded.shape[0],
                                             imageToTest_padded.shape[1]))
            #net.forward() # dry run
            self.net.blobs['data'].data[...] = np.transpose(
                np.float32(imageToTest_padded[:, :, :, np.newaxis]),
                (3, 2, 0, 1)) / 256 - 0.5
            start_time = time.time()
            output_blobs = self.net.forward()
            print('At scale %d, The CNN took %.2f ms.' %
                  (m, 1000 * (time.time() - start_time)))

            # extract outputs, resize, and remove padding
            heatmap = np.transpose(
                np.squeeze(self.net.blobs[output_blobs.keys()[1]].data),
                (1, 2, 0))  # output 1 is heatmaps
            heatmap = cv.resize(heatmap, (0, 0),
                                fx=model['stride'],
                                fy=model['stride'],
                                interpolation=cv.INTER_CUBIC)
            heatmap = heatmap[:imageToTest_padded.shape[0] -
                              pad[2], :imageToTest_padded.shape[1] - pad[3], :]
            heatmap = cv.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]),
                                interpolation=cv.INTER_CUBIC)

            paf = np.transpose(
                np.squeeze(self.net.blobs[output_blobs.keys()[0]].data),
                (1, 2, 0))  # output 0 is PAFs
            paf = cv.resize(paf, (0, 0),
                            fx=model['stride'],
                            fy=model['stride'],
                            interpolation=cv.INTER_CUBIC)
            paf = paf[:imageToTest_padded.shape[0] -
                      pad[2], :imageToTest_padded.shape[1] - pad[3], :]
            paf = cv.resize(paf, (oriImg.shape[1], oriImg.shape[0]),
                            interpolation=cv.INTER_CUBIC)

            # visualization
            for k in range(14):
                '''
                axarr2[k,m].imshow(oriImg[:,:,[2,1,0]])
                ax2 = axarr2[k,m].imshow(heatmap[:,:,k], alpha=.5) # right wrist
                axarr2[k,m].set_title('Heatmaps (Rwri): scale %d' % m)
                '''
                '''
                plt.imshow(oriImg[:,:,[2,1,0]])
                plt.imshow(heatmap[:,:,k], alpha=.5) # right wrist
                plt.savefig('{}_{}.png'.format(k, m))
                '''
            '''
            axarr3.flat[m].imshow(oriImg[:,:,[2,1,0]])
            ax3x = axarr3.flat[m].imshow(paf[:,:,16], alpha=.5) # right elbow
            axarr3.flat[m].set_title('PAFs (x comp. of Rwri to Relb): scale %d' % m)
            axarr3.flat[len(multiplier) + m].imshow(oriImg[:,:,[2,1,0]])
            ax3y = axarr3.flat[len(multiplier) + m].imshow(paf[:,:,17], alpha=.5) # right wrist
            axarr3.flat[len(multiplier) + m].set_title('PAFs (y comp. of Relb to Rwri): scale %d' % m)
            '''

            heatmap_avg = heatmap_avg + heatmap / len(multiplier)
            paf_avg = paf_avg + paf / len(multiplier)
        '''
        for i in range(14):
            plt.imshow(oriImg[:,:,[2,1,0]])
            plt.imshow(heatmap_avg[:,:,i], alpha=.5)
            plt.savefig(str(i))
        '''
        '''
        for i in range(13):
            U = paf_avg[:,:,i*2] * -1
            V = paf_avg[:,:,i*2+1]
            X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0]))
            M = np.zeros(U.shape, dtype='bool')
            M[U**2 + V**2 < 0.5 * 0.5] = True
            U = ma.masked_array(U, mask=M)
            V = ma.masked_array(V, mask=M)

        # 1
            plt.figure()
            plt.imshow(oriImg[:,:,[2,1,0]], alpha = .5)
            s = 5
            Q = plt.quiver(X[::s,::s], Y[::s,::s], U[::s,::s], V[::s,::s], 
                           scale=50, headaxislength=4, alpha=.5, width=0.001, color='r')
            plt.savefig('_{}.png'.format(i))
        '''
        '''
            fig = matplotlib.pyplot.gcf()
            fig.set_size_inches(20, 20)
        '''

        print heatmap_avg.shape

        #plt.imshow(heatmap_avg[:,:,2])
        all_peaks = []
        peak_counter = 0

        for part in range(15 - 1):
            x_list = []
            y_list = []
            map_ori = heatmap_avg[:, :, part]
            map = gaussian_filter(map_ori, sigma=3)

            map_left = np.zeros(map.shape)
            map_left[1:, :] = map[:-1, :]
            map_right = np.zeros(map.shape)
            map_right[:-1, :] = map[1:, :]
            map_up = np.zeros(map.shape)
            map_up[:, 1:] = map[:, :-1]
            map_down = np.zeros(map.shape)
            map_down[:, :-1] = map[:, 1:]

            peaks_binary = np.logical_and.reduce(
                (map >= map_left, map >= map_right, map >= map_up,
                 map >= map_down, map > param['thre1']))
            peaks = zip(
                np.nonzero(peaks_binary)[1],
                np.nonzero(peaks_binary)[0])  # note reverse
            peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
            id = range(peak_counter, peak_counter + len(peaks))
            peaks_with_score_and_id = [
                peaks_with_score[i] + (id[i], ) for i in range(len(id))
            ]

            all_peaks.append(peaks_with_score_and_id)
            peak_counter += len(peaks)

# find connection in the specified sequence, center 29 is in the position 15
        limbSeq = [[13, 14], [14, 4], [4, 5], [5, 6], [14, 1], [1, 2], [2, 3],
                   [14, 10], [10, 11], [11, 12], [14, 7], [7, 8], [8, 9]]

        # the middle joints heatmap correpondence
        mapIdx = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13],
                  [14, 15], [16, 17], [18, 19], [20, 21], [22, 23], [24, 25]]

        connection_all = []
        special_k = []
        mid_num = 10

        for k in range(len(mapIdx)):
            score_mid = paf_avg[:, :, [x for x in mapIdx[k]]]
            candA = all_peaks[limbSeq[k][0] - 1]
            candB = all_peaks[limbSeq[k][1] - 1]
            nA = len(candA)
            nB = len(candB)
            indexA, indexB = limbSeq[k]
            if (nA != 0 and nB != 0):
                connection_candidate = []
                for i in range(nA):
                    for j in range(nB):
                        vec = np.subtract(candB[j][:2], candA[i][:2])
                        norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
                        vec = np.divide(vec, norm)

                        startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
                                       np.linspace(candA[i][1], candB[j][1], num=mid_num))

                        vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
                                          for I in range(len(startend))])
                        vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
                                          for I in range(len(startend))])

                        score_midpts = np.multiply(
                            vec_x, vec[0]) + np.multiply(vec_y, vec[1])
                        score_with_dist_prior = sum(score_midpts) / len(
                            score_midpts) + min(
                                0.5 * oriImg.shape[0] / norm - 1, 0)
                        criterion1 = len(
                            np.nonzero(score_midpts > param['thre2'])
                            [0]) > 0.8 * len(score_midpts)
                        criterion2 = score_with_dist_prior > 0
                        if criterion1 and criterion2:
                            connection_candidate.append([
                                i, j, score_with_dist_prior,
                                score_with_dist_prior + candA[i][2] +
                                candB[j][2]
                            ])

                connection_candidate = sorted(connection_candidate,
                                              key=lambda x: x[2],
                                              reverse=True)
                connection = np.zeros((0, 5))
                for c in range(len(connection_candidate)):
                    i, j, s = connection_candidate[c][0:3]
                    if (i not in connection[:, 3]
                            and j not in connection[:, 4]):
                        connection = np.vstack(
                            [connection, [candA[i][3], candB[j][3], s, i, j]])
                        if (len(connection) >= min(nA, nB)):
                            break

                connection_all.append(connection)
            else:
                special_k.append(k)
                connection_all.append([])

# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
        subset = -1 * np.ones((0, 20))
        candidate = np.array(
            [item for sublist in all_peaks for item in sublist])

        for k in range(len(mapIdx)):
            if k not in special_k:
                partAs = connection_all[k][:, 0]
                partBs = connection_all[k][:, 1]
                indexA, indexB = np.array(limbSeq[k]) - 1

                for i in range(len(connection_all[k])):  #= 1:size(temp,1)
                    found = 0
                    subset_idx = [-1, -1]
                    for j in range(len(subset)):  #1:size(subset,1):
                        if subset[j][indexA] == partAs[i] or subset[j][
                                indexB] == partBs[i]:
                            subset_idx[found] = j
                            found += 1

                    if found == 1:
                        j = subset_idx[0]
                        if (subset[j][indexB] != partBs[i]):
                            subset[j][indexB] = partBs[i]
                            subset[j][-1] += 1
                            subset[j][-2] += candidate[
                                partBs[i].astype(int),
                                2] + connection_all[k][i][2]
                    elif found == 2:  # if found 2 and disjoint, merge them
                        j1, j2 = subset_idx
                        print "found = 2"
                        membership = ((subset[j1] >= 0).astype(int) +
                                      (subset[j2] >= 0).astype(int))[:-2]
                        if len(np.nonzero(membership == 2)[0]) == 0:  #merge
                            subset[j1][:-2] += (subset[j2][:-2] + 1)
                            subset[j1][-2:] += subset[j2][-2:]
                            subset[j1][-2] += connection_all[k][i][2]
                            subset = np.delete(subset, j2, 0)
                        else:  # as like found == 1
                            subset[j1][indexB] = partBs[i]
                            subset[j1][-1] += 1
                            subset[j1][-2] += candidate[
                                partBs[i].astype(int),
                                2] + connection_all[k][i][2]

                    # if find no partA in the subset, create a new subset
                    elif not found and k < 17:
                        row = -1 * np.ones(20)
                        row[indexA] = partAs[i]
                        row[indexB] = partBs[i]
                        row[-1] = 2
                        row[-2] = sum(
                            candidate[connection_all[k][i, :2].astype(int),
                                      2]) + connection_all[k][i][2]
                        subset = np.vstack([subset, row])


# delete some rows of subset which has few parts occur
        deleteIdx = []
        for i in range(len(subset)):
            if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
                deleteIdx.append(i)
        subset = np.delete(subset, deleteIdx, axis=0)

        # visualize
        b = [255, 0, 0]
        r = [0, 0, 255]
        g = [0, 255, 0]
        colors = [b, b, b, r, r, r, b, b, b, r, r, r, g, g]
        colors2 = [g, r, r, r, b, b, b, r, r, r, b, b, b]
        '''
        [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
                  [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
                  [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
        '''

        cmap = matplotlib.cm.get_cmap('hsv')

        canvas = cv.imread(test_image)  # B,G,R order

        for i in range(14):
            rgba = np.array(cmap(1 - i / 18. - 1. / 36))
            rgba[0:3] *= 255
            for j in range(len(all_peaks[i])):
                cv.circle(canvas,
                          all_peaks[i][j][0:2],
                          4,
                          colors[i],
                          thickness=-1)

        to_plot = cv.addWeighted(oriImg, 0.3, canvas, 0.7, 0)
        plt.imshow(to_plot[:, :, [2, 1, 0]])
        fig = matplotlib.pyplot.gcf()
        fig.set_size_inches(12, 12)

        # visualize 2
        stickwidth = 4

        for i in range(13):
            for n in range(len(subset)):
                index = subset[n][np.array(limbSeq[i]) - 1]
                if -1 in index:
                    continue
                cur_canvas = canvas.copy()
                Y = candidate[index.astype(int), 0]
                X = candidate[index.astype(int), 1]
                mX = np.mean(X)
                mY = np.mean(Y)
                length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5
                angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
                polygon = cv.ellipse2Poly((int(mY), int(mX)),
                                          (int(length / 2), stickwidth),
                                          int(angle), 0, 360, 1)
                cv.fillConvexPoly(cur_canvas, polygon, colors2[i % 13])
                canvas = cv.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)

        plt.imshow(canvas[:, :, [2, 1, 0]])

        fig = matplotlib.pyplot.gcf()
        fig.set_size_inches(12, 12)
        plt.savefig('predict_{}Crop'.format(idx))