def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active:
draw_points_norm([self.smooth_pos],size=15,color=RGBA(1.,1.,0.,.5))
if self.active and self.detected:
for e in self.candidate_ellipses:
pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
(int(e[1][0]/2),int(e[1][1]/2)),
int(e[-1]),0,360,15)
draw_polyline(pts,color=RGBA(0.,1.,0,1.))
# lets draw an indicator on the autostop count
e = self.candidate_ellipses[3]
pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
(int(e[1][0]/2),int(e[1][1]/2)),
int(e[-1]),0,360,360/self.auto_stop_max)
indicator = [e[0]] + pts[self.auto_stop:].tolist() + [e[0]]
draw_polyline(indicator,color=RGBA(8.,0.1,0.1,.8),line_type=GL_POLYGON)
else:
pass
def create_mask(self, shape):
MotionTrackedBodypart.create_mask(self, shape)
self.wfnRoiMaskedPolarCropped = None
if (self.mask.xMin is not None):
self.i_0 = self.params['gui'][self.name]['hinge']['y'] - self.roi[1]
self.j_0 = self.params['gui'][self.name]['hinge']['x'] - self.roi[0]
# Args for the ellipse calls.
x = int(self.params['gui'][self.name]['hinge']['x'])
y = int(self.params['gui'][self.name]['hinge']['y'])
r_outer = int(np.ceil(self.params['gui'][self.name]['radius_outer']))
r_inner = int(np.floor(self.params['gui'][self.name]['radius_inner']))-1
hi = int(np.ceil(np.rad2deg(self.angle_hi_i)))
lo = int(np.floor(np.rad2deg(self.angle_lo_i)))
# Find where the mask might be clipped. First, draw an unclipped ellipse.
delta = 1
ptsUnclipped = cv2.ellipse2Poly((x, y), (r_outer, r_outer), 0, hi, lo, delta)
ptsUnclipped = np.append(ptsUnclipped, cv2.ellipse2Poly((x, y), (r_inner, r_inner), 0, hi, lo, delta), 0)
#ptsUnclipped = np.append(ptsUnclipped, [list of line1 pixels connecting the two arcs], 0)
#ptsUnclipped = np.append(ptsUnclipped, [list of line2 pixels connecting the two arcs], 0)
# These are the unclipped locations.
minUnclipped = ptsUnclipped.min(0)
maxUnclipped = ptsUnclipped.max(0)
xMinUnclipped = minUnclipped[0]
yMinUnclipped = minUnclipped[1]
xMaxUnclipped = maxUnclipped[0]+1
yMaxUnclipped = maxUnclipped[1]+1
# Compare unclipped with the as-drawn locations.
xClip0 = self.mask.xMin - xMinUnclipped
yClip0 = self.mask.yMin - yMinUnclipped
xClip1 = xMaxUnclipped - self.mask.xMax
yClip1 = yMaxUnclipped - self.mask.yMax
roiClipped = np.array([xClip0, yClip0, xClip1, yClip1])
(i_n, j_n) = shape[:2]
# Determine how much of the bottom of the polar image to trim off (i.e. rClip) based on if the ellipse is partially offimage.
(rClip0, rClip1, rClip2, rClip3) = (1.0, 1.0, 1.0, 1.0)
if (roiClipped[0]>0): # Left
rClip0 = 1.0 - (float(roiClipped[0])/float(r_outer-r_inner))#self.j_0))
if (roiClipped[1]>0): # Top
rClip1 = 1.0 - (float(roiClipped[1])/float(r_outer-r_inner))#self.i_0))
if (roiClipped[2]>0): # Right
rClip2 = 1.0 - (float(roiClipped[2])/float(r_outer-r_inner))#j_n-self.j_0))
if (roiClipped[3]>0): # Bottom
rClip3 = 1.0 - (float(roiClipped[3])/float(r_outer-r_inner))#i_n-self.i_0))
self.rClip = np.min([rClip0, rClip1, rClip2, rClip3])
def drawOn(self, canvas, color=(0, 0, 255), degreeDelta=20):
points = cv2.ellipse2Poly(tuple(map(int, self.position)),
tuple(map(int, self.axes)),
self.angle*180/math.pi,
0, 360,
degreeDelta)
cv2.fillConvexPoly(canvas, points, color, cv2.CV_AA)
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
# debug mode within world will show green ellipses around detected ellipses
if self.active:
for marker in self.markers:
e = marker["ellipses"][-1] # outermost ellipse
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, 1, RGBA(0.0, 1.0, 0.0, 1.0))
if len(self.markers) > 1:
draw_polyline(
pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON
)
def draw_hand(full_img, joint_coords, is_loss_track):
if is_loss_track:
joint_coords = FLAGS.default_hand
# Plot joints
for joint_num in range(FLAGS.num_of_joints):
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
joint_color = list(map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(full_img, center=(int(joint_coords[joint_num][1]), int(joint_coords[joint_num][0])), radius=3,
color=joint_color, thickness=-1)
else:
joint_color = list(map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(full_img, center=(int(joint_coords[joint_num][1]), int(joint_coords[joint_num][0])), radius=3,
color=joint_color, thickness=-1)
# Plot limbs
for limb_num in range(len(FLAGS.limbs)):
x1 = int(joint_coords[int(FLAGS.limbs[limb_num][0])][0])
y1 = int(joint_coords[int(FLAGS.limbs[limb_num][0])][1])
x2 = int(joint_coords[int(FLAGS.limbs[limb_num][1])][0])
y2 = int(joint_coords[int(FLAGS.limbs[limb_num][1])][1])
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
if length < 150 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
color_code_num = limb_num // 4
limb_color = list(map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.fillConvexPoly(full_img, polygon, color=limb_color)
def _visualize(self, img, joint_cands_indices, all_peaks, candidate):
cmap = matplotlib.cm.get_cmap('hsv')
for i in range(len(self.index2limbname)-1):
rgba = np.array(cmap(1 - i / 18. - 1. / 36))
rgba[0:3] *= 255
for j in range(len(all_peaks[i])):
cv2.circle(img, (int(all_peaks[i][j][0]), int(
all_peaks[i][j][1])), 4, self.colors[i], thickness=-1)
stickwidth = 4
for i in range(len(self.index2limbname) - 2):
for joint_cand_indices in joint_cands_indices:
index = joint_cand_indices[np.array(self.limb_sequence[i],
dtype=np.int32) - 1]
if -1 in index:
continue
cur_img = img.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(
length / 2), stickwidth), int(angle), 0, 360, 1)
cv2.fillConvexPoly(cur_img, polygon, self.colors[i])
img = cv2.addWeighted(img, 0.4, cur_img, 0.6, 0)
return img
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active and self.detected:
for e in self.candidate_ellipses:
pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
(int(e[1][0]/2),int(e[1][1]/2)),
int(e[-1]),0,360,15)
draw_gl_polyline(pts,(0.,1.,0,1.))
else:
pass
if self._window:
self.gl_display_in_window()
if not self.active and self.error_lines is not None:
draw_gl_polyline_norm(self.error_lines,(1.,0.5,0.,.5),type='Lines')
draw_gl_points_norm(self.error_lines[1::2],color=(.0,0.5,0.5,.5),size=3)
draw_gl_points_norm(self.error_lines[0::2],color=(.5,0.0,0.0,.5),size=3)
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active:
# draw the largest ellipse of all detected markers
for marker in self.markers:
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
15,
)
draw_polyline(pts, color=RGBA(0.0, 1.0, 0, 1.0))
if len(self.markers) > 1:
draw_polyline(
pts, 1, RGBA(1.0, 0.0, 0.0, 0.5), line_type=gl.GL_POLYGON
)
# draw indicator on the stop marker(s)
if self.auto_stop:
for marker in self.markers:
if marker["marker_type"] == "Stop":
e = marker["ellipses"][-1]
pts = cv2.ellipse2Poly(
(int(e[0][0]), int(e[0][1])),
(int(e[1][0] / 2), int(e[1][1] / 2)),
int(e[-1]),
0,
360,
360 // self.auto_stop_max,
)
indicator = [e[0]] + pts[self.auto_stop :].tolist() + [e[0]]
draw_polyline(
indicator,
color=RGBA(8.0, 0.1, 0.1, 0.8),
line_type=gl.GL_POLYGON,
)
def draw_all_ellipses(self):
# draws all ellipses in self.ellipses.
glPushMatrix()
for ellipse in self.ellipses[-10:]:
glColor3f(0.0, 1.0, 0.0) #set color to green
pts = cv2.ellipse2Poly( (int(ellipse.center[0]),int(ellipse.center[1])),
(int(ellipse.major_radius),int(ellipse.minor_radius)),
int(ellipse.angle*180/scipy.pi),0,360,15)
draw_polyline(pts,2,color = RGBA(0,1,1,.5))
glPopMatrix()
def polytest(x,y,rx,ry,rw,rh,rang):
points=cv2.ellipse2Poly(
(rx,ry),
axes=(rw/2,rh/2),
angle=rang,
arcStart=0,
arcEnd=360,
delta=3
)
return cv2.pointPolygonTest(np.array(points), (x,y), measureDist=1)
def draw_circle(self, frame, pos,r,c):
''' function that is called by draw_marker exclusively '''
pts = cv2.ellipse2Poly(tuple(pos),(r,r),0,0,360,10)
draw_gl_polyline(pts,c,'Polygon')
world_size = self.world_size
window_size = glfwGetWindowSize(self._window)
ratio = [float(world_size[0])/window_size[0], float(world_size[1])/window_size[1]]
pos_0 = ratio[0] * pos[0] * .75 + world_size[0] * (1 - .75)/2
pos_1 = ratio[1] * pos[1] * .75 + world_size[1] * (1 - .75)/2
pos_1 = int(pos_0), int(pos_1)
c = cv2.cv.CV_RGB(c[0]*255, c[1]*255, c[2]*255)
cv2.ellipse(frame, tuple(pos_1),(r,r),0,0,360, c)
def get_each(self, stimulus, history_length = 0, approx = False):
ovImg = np.zeros((stimulus.height, stimulus.width,3),'uint8')
colors = utils.autumn(len(self.gazes))
gaze_coords = self.getGazeMovieCoords(stimulus, history_length, approx)
gaze_coords = sorted(gaze_coords,key=operator.itemgetter(3),reverse=True)
for (i, x, y, prio) in gaze_coords:
if prio == 1:
cv2.circle(ovImg, (int(x*stimulus.width),int(y*stimulus.height)), 15, colors[int(i)])
pts = cv2.ellipse2Poly((int(x*stimulus.width),int(y*stimulus.height)),(2,2),0,0,360,int(360/6))
opacity = float(history_length-prio+2) / (history_length+1)
cv2.fillConvexPoly(ovImg, pts, (int(colors[int(i)][0]*opacity), int(colors[int(i)][1]*opacity), int(colors[int(i)][2]*opacity)))
return ovImg
def ellipse2Rect(params):
rad_x = params[0]
rad_y = params[1]
angle = params[2] * 180.0 / pi
center_x = params[3]
center_y = params[4]
pts = cv.ellipse2Poly((int(center_x), int(center_y)), (int(rad_x), int(rad_y)),
int(angle), 0, 360, 10)
rect = cv.boundingRect(pts)
left = rect[0]
top = rect[1]
right = rect[0] + rect[2]
bottom = rect[1] + rect[3]
return left, top, right, bottom
def draw_limbs_2d(img, joints_2d, limb_parents):
for limb_num in range(len(limb_parents)-1):
x1 = joints_2d[limb_num, 0]
y1 = joints_2d[limb_num, 1]
x2 = joints_2d[limb_parents[limb_num], 0]
y2 = joints_2d[limb_parents[limb_num], 1]
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
# if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
cv2.fillConvexPoly(img, polygon, color=(0,255,0))
return img
def parse_json(json_file, image_file):
canvas = cv2.imread(image_file) # B,G,R order
with open(json_file) as infile:
info = json.load(infile)
pose_list = list()
count = 1
for item in info['persons']:
count += 1
pose_list.append(item['keypoints'])
pose_array = np.array(pose_list)
print pose_array.shape
cmap = matplotlib.cm.get_cmap('hsv')
for i in range(18):
rgba = np.array(cmap(1 - i/18. - 1./36))
rgba[0:3] *= 255
for j in range(pose_array.shape[0]):
if pose_array[j][i][2] == 2:
continue
cv2.circle(canvas, (int(pose_array[j][i][0]), int(pose_array[j][i][1])), 4, colors[i], thickness=-1)
cv2.imshow("main", canvas)
cv2.waitKey()
for i in range(17):
for n in range(pose_array.shape[0]):
Y = pose_array[n][np.array(limbSeq[i])-1][:,0]
X = pose_array[n][np.array(limbSeq[i])-1][:,1]
if 2. in pose_array[n][np.array(limbSeq[i])-1][:,2]:
continue
cur_canvas = canvas.copy()
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly((int(mY),int(mX)), (int(length/2), 4), int(angle), 0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
cv2.imshow("main", canvas)
cv2.waitKey()
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active and self.detected:
for e in self.candidate_ellipses:
pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
(int(e[1][0]/2),int(e[1][1]/2)),
int(e[-1]),0,360,15)
draw_gl_polyline(pts,(0.,1.,0,1.))
else:
pass
def link_key_point(img_canvas, candidate, subset, stickwidth=4):
for i in range(17):
for n in range(len(subset)):
index = subset[n][np.array(limb_seq[i]) - 1]
if -1 in index:
continue
cur_canvas = img_canvas.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
img_canvas = cv2.addWeighted(img_canvas, 0.4, cur_canvas, 0.6, 0)
return img_canvas
def getPoly(self, typ, pts):
if typ == 0:
return []
elif typ == 1:
x = pts[:,0]
y = pts[:,1]
x.sort()
y.sort()
return np.array([[x[0],y[0]],[x[0],y[1]],[x[1],y[1]],[x[1],y[0]]])
elif typ == 2:
r = int(np.linalg.norm(pts[0,:] - pts[1,:]))
return cv2.ellipse2Poly((pts[0,0],pts[0,1]), (r,r), 0, 0, 360, 10)
elif typ == 3:
return pts[0:-2,:]
elif typ == 4:
print "Warning: Place->getPoly(): typ=4 (ellipse) not implemented yet"
return None
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
if self.active and self.detected:
draw_points_norm([self.smooth_pos],size=15,color=RGBA(1.,1.,0.,.5))
for e in self.candidate_ellipses:
pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
(int(e[1][0]/2),int(e[1][1]/2)),
int(e[-1]),0,360,15)
draw_polyline(pts,color=RGBA(0.,1.,0,1.))
else:
pass
def visualize(self,all_peaks, image_path, subset, candidate, dest_points, dest_skeleton):
limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], [10, 11], [2, 12], [12, 13],
[13, 14], [2, 1], [1, 15], [15, 17], [1, 16], [16, 18], [3, 17], [6, 18]]
colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0],
[0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255],
[170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
cmap = matplotlib.cm.get_cmap('hsv')
oriImg = cv.imread(image_path)
canvas = cv.imread(image_path)
for i in range(18):
rgba = np.array(cmap(1 - i / 18. - 1. / 36))
rgba[0:3] *= 255
for j in range(len(all_peaks[i])):
cv.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
to_plot = cv.addWeighted(oriImg, 0.3, canvas, 0.7, 0)
#cv.imwrite(dest_points, to_plot)
#plt.imshow(to_plot[:, :, [2, 1, 0]])
#plt.show()
stickwidth = 4
for i in range(17):
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[i]) - 1]
if -1 in index:
continue
cur_canvas = canvas.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
cv.fillConvexPoly(cur_canvas, polygon, colors[i])
canvas = cv.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
cv.imwrite(dest_skeleton, canvas)
def __link_key_point(self, img_canvas, candidate, subset):
for i in range(self.configer.get('data', 'num_keypoints')-1):
for n in range(len(subset)):
index = subset[n][np.array(self.configer.get('details', 'limb_seq')[i]) - 1]
if -1 in index:
continue
cur_canvas = img_canvas.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly((int(mY), int(mX)),
(int(length / 2),
self.configer.get('vis', 'stick_width')), int(angle), 0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, self.configer.get('details', 'color_list')[i])
img_canvas = cv2.addWeighted(img_canvas, 0.4, cur_canvas, 0.6, 0)
return img_canvas
def gl_display(self):
"""
use gl calls to render
at least:
the published position of the reference
better:
show the detected postion even if not published
"""
# debug mode within world will show green ellipses around detected ellipses
if self.active and self.detected:
for e in self.candidate_ellipses:
pts = cv2.ellipse2Poly( (int(e[0][0]),int(e[0][1])),
(int(e[1][0]/2),int(e[1][1]/2)),
int(e[-1]),0,360,15)
draw_polyline(pts,1,RGBA(0.,1.,0.,1.))
else:
pass
if self._window:
self.gl_display_in_window()
def get_withinring(ringpolyTVArray, timepoints, x, y):
rx = ringpolyTVArray[:,0].astype(np.int)
ry = ringpolyTVArray[:,1].astype(np.int)
rw = ringpolyTVArray[:,2].astype(np.int)
rh = ringpolyTVArray[:,3].astype(np.int)
rang = ringpolyTVArray[:,4].astype(np.int)
# poly test
peaks_within = []
for p in timepoints:
points=cv2.ellipse2Poly(
(rx[p],ry[p]),
axes=(rw[p]/2,rh[p]/2),
angle=rang[p],
arcStart=0,
arcEnd=360,
delta=3
)
inout = cv2.pointPolygonTest(np.array(points), (x[p],y[p]), measureDist=1)
if inout > 0:
peaks_within.append(p)
return peaks_within
def draw_circle(pos,r,c):
pts = cv2.ellipse2Poly(tuple(pos),(r,r),0,0,360,10)
draw_gl_polyline(pts,c,'Polygon')
def renderface(facepts, canvas, disp=False, threshold = 0.2, smalldot = 2): if disp: color = tuple([255, 255, 255]) else: color = tuple([0, 0, 0]) avecons = sum(facepts[2:len(facepts):3]) / 70.0 if avecons < threshold: return canvas i = 0 while i < 210: confidence = facepts[i+2] if confidence > 0: cv.circle(canvas, (int(facepts[i]), int(facepts[i+1])), smalldot, color, thickness=-1) i += 3 if disp: #graph the lines between points stickwidth = 1 linearSeq = [range(0, 16+1), range(17, 21+1), range(22, 26+1), range(27, 30+1), range(31, 35+1)] circularSeq = [range(36, 41+1), range(42, 47+1), range(48, 59+1), range(60, 67)] for line in linearSeq: for step in line: if step != line[len(line) - 1]: firstlimb_ind = step secondlimb_ind = step + 1 if (facepts[3*firstlimb_ind + 2] > 0) and (facepts[3*secondlimb_ind + 2] > 0): cur_canvas = canvas.copy() Y = [facepts[3*firstlimb_ind], facepts[3*secondlimb_ind]] X = [facepts[3*firstlimb_ind + 1], facepts[3*secondlimb_ind + 1]] mX = np.mean(X) mY = np.mean(Y) length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) polygon = cv.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1) cv.fillConvexPoly(cur_canvas, polygon, color) canvas = cv.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0) for circle in circularSeq: for step in circle: if step == circle[len(circle) - 1]: firstlimb_ind = step secondlimb_ind = circle[0] else: firstlimb_ind = step secondlimb_ind = step + 1 if (facepts[3*firstlimb_ind + 2] > 0) and (facepts[3*secondlimb_ind + 2] > 0): cur_canvas = canvas.copy() Y = [facepts[3*firstlimb_ind], facepts[3*secondlimb_ind]] X = [facepts[3*firstlimb_ind + 1], facepts[3*secondlimb_ind + 1]] mX = np.mean(X) mY = np.mean(Y) length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) polygon = cv.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1) cv.fillConvexPoly(cur_canvas, polygon, color) canvas = cv.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0) return canvas
def do_work(test_image,save_name):
oriImg = cv2.imread(test_image) # B,G,R order
multiplier = [x * model['boxsize'] / oriImg.shape[0] for x in param['scale_search']]
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
print 'len of multipler'
print len(multiplier)
for m in range(len(multiplier)):
scale = multiplier[m]
print scale
imageToTest = cv2.resize(oriImg, (0,0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.padRightDownCorner(imageToTest, model['stride'], model['padValue'])
print imageToTest_padded.shape
net.blobs['data'].reshape(*(1, 3, imageToTest_padded.shape[0], imageToTest_padded.shape[1]))
#net.forward() # dry run
net.blobs['data'].data[...] = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,2,0,1))/256 - 0.5;
start_time = time.time()
output_blobs = net.forward()
print('At scale %d, The CNN took %.2f ms.' % (m, 1000 * (time.time() - start_time)))
start_time = time.time()
# extract outputs, resize, and remove padding
heatmap = np.transpose(np.squeeze(net.blobs[output_blobs.keys()[1]].data), (1,2,0)) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0,0), fx=model['stride'], fy=model['stride'], interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
paf = np.transpose(np.squeeze(net.blobs[output_blobs.keys()[0]].data), (1,2,0)) # output 0 is PAFs
paf = cv2.resize(paf, (0,0), fx=model['stride'], fy=model['stride'], interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
print('2 At scale %d, The CNN took %.2f ms.' % (m, 1000 * (time.time() - start_time)))
start_time = time.time()
all_peaks = []
peak_counter = 0
for part in range(19-1):
x_list = []
y_list = []
map_ori = heatmap_avg[:,:,part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:,:] = map[:-1,:]
map_right = np.zeros(map.shape)
map_right[:-1,:] = map[1:,:]
map_up = np.zeros(map.shape)
map_up[:,1:] = map[:,:-1]
map_down = np.zeros(map.shape)
map_down[:,:-1] = map[:,1:]
peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param['thre1']))
peaks = zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]) # note reverse
peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf_avg[:,:,[x-19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0]-1]
candB = all_peaks[limbSeq[k][1]-1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if(nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
norm = math.sqrt(vec[0]*vec[0] + vec[1]*vec[1])
vec = np.divide(vec, norm)
startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num))
vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
score_with_dist_prior = sum(score_midpts)/len(score_midpts) + min(0.5*oriImg.shape[0]/norm-1, 0)
criterion1 = len(np.nonzero(score_midpts > param['thre2'])[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior+candA[i][2]+candB[j][2]])
connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
connection = np.zeros((0,5))
for c in range(len(connection_candidate)):
i,j,s = connection_candidate[c][0:3]
if(i not in connection[:,3] and j not in connection[:,4]):
connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
if(len(connection) >= min(nA, nB)):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:,0]
partBs = connection_all[k][:,1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])): #= 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): #1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if(subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
print "found = 2"
membership = ((subset[j1]>=0).astype(int) + (subset[j2]>=0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: #merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(candidate[connection_all[k][i,:2].astype(int), 2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = [];
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2]/subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
cmap = matplotlib.cm.get_cmap('hsv')
canvas = cv2.imread(test_image) # B,G,R order
for i in range(18):
rgba = np.array(cmap(1 - i/18. - 1./36))
rgba[0:3] *= 255
for j in range(len(all_peaks[i])):
cv2.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
# visualize 2
stickwidth = 4
for i in range(17):
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[i])-1]
if -1 in index:
continue
cur_canvas = canvas.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
cv2.imwrite(save_name,canvas)
print('3 At scale %d, The CNN took %.2f ms.' % (m, 1000 * (time.time() - start_time)))
def extend_joint_area(self, joint, radius=65):
x = int(round(joint[0])
y = int(round(joint[1])
left = x - radius
top = y - radius
right = x + radius
bottom = y + radius
#cropped_img = image[y-radius:y+radius, x-radius:x+radius]
return list(zip(left, top, right, bottom))
def calculate_HOI(self):
self.relations = []
joints = self.pose.get_pose()
if not check_for_person(joints):
print("No person found")
object_data = self.object.predict() #Image should be saved elsewhere
#Each check returns the object list any objects that were found removed from the next check (stops items being held and eaten at the same time)
object_data = check_for_face_interactions(object_data, joints)
object_data = check_for_hand_interactions(object_data, joints)
object_data = check_for_misc_interactions(object_data, joints)
print("Unused objects: " + object_data)
return self.relations
class Pose_Mirroring():
def __init__(self, pose, model_dir="D:/RobotMaster/Models", model_name="InceptionV3.hdf5"):
self.pose = pose
self.model = init_custom_pose_model(model_dir, model_name)
self.part_pairs = [1,2, 1,5, 2,3, 3,4, 5,6, 6,7, 1,0, 0,15, 15,17]
self.colours = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
[0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
[170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
pass
#Done - untested
def init_custom_pose_model(self, dir, name):
try:
model = load_model(os.path.join(dir, name))
return model
except:
raise Exception("Model not found")
def get_pose(self, frame):
keypoints = pose.get_pose(frame)
num_people = len(keypoints)
if num_people == 0:
print("No people found")
return None
#Only run on the first skeleton found, not multiple people
frame = self.draw_person(frame, keypoints[0])
label = self.predict(frame)
label = self.label_to_name("D:/RobotMaster/res/legend.txt", label
return label
def label_to_name(self, legend, label_val):
with open(legend, 'r') as f:
lines = f.readlines()
return lines[int(label_num].split(",")[1]
def predict(self, img, img_target_size=299):
img = cv2.resize(img, (img_target_size, img_target_size))
x = np.expand_dims(img, axis=0)
x = preprocess_input(x.astype(float))
pred = self.model.predict(x)
pred = pred.tolist()
pred = pred[0]
return pred.index(max(pred))
def draw_person(self, img, person):
partial_joints = [0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 17, 18]
upper_skeleton_img = np.zeros(img.shape[0], img.shape[1], 3), np.unit8)
counter = 0
for data in person:
x, y, score = data
if x > 0 and y > 0:
if counter in partial_joints:
cv2.cricle(upper_skeleton_img, (round(x), round(y)), 7, (0, 0, 255), -1)
counter += 1
upper_skeleton_img = visualise_person(upper_skeleton_img, person)
cropped_img = crop_person(upper_skeleton_img, person, partial_joints)
return cropped_img
def crop_person(self, img, person, upper_indexs):
minx = 99999
miny = 99999
maxx = 0
maxy = 0
counter = 0
for joint in person:
x, y, score = joint
if counter in partial_joints:
if x > 0
if round(x) > maxx:
maxx = x + 10
if round(x) < minx:
minx = x - 10
if round(y) > maxy:
maxy = y + 10
if round(y) < miny:
miny = y - 10
counter += 1
cropped = img[int(round(miny)):int(round(maxy)), int(round(minx)):int(round(maxx))]
return cropped
def visualise_person(self, img, person):
pairs = self.part_pairs
stickwidth = 4
cur_img = img.copy()
counter = 0;
for i in range(0, len(pairs),2):
if person[pairs[i],0] > 0 and person[pairs[i+1],0] > 0:
Y = [person[pairs[i],0], person[pairs[i+1],0]]
X = [person[pairs[i],1], person[pairs[i+1],1]]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
cv2.fillConvexPoly(cur_img, polygon, colours[counter])
counter = counter + 1
img = cv2.addWeighted(img, 0.4, cur_img, 0.6, 0)
return img
class Age_Gender():
def __init__(self, dir="D:/NAORobot/RossProject/"):
self.bat_file_dir = dir
pass
def start(self):
Popen(os.path.join(self.dir, "CompleteBuild.bat"))
class Secret_Project():
def __init__(self):
self.lower_blue = np.array([0, 102, 255])
self.upper_blue = np.array([0, 0, 204])
pass
def apply_mask(self, img):
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(hsv, self.lower_blue, self.upper_blue)
res = cv2.bitwise_and(img,img, mask=mask)
kernel = np.ones((15,15),np.float32)/225
smoothed = cv2.filter2D(res,-1,kernel)
_, secret = cv2.threshold(smoothed, 30, 255, cv2.THRESH_BINARY)
cv2.imshow("average", smoothed)
cv2.imshow("secret", secret)
if cv2.waitKey(1) & 0xFF == 27:
cv2.destroyAllWindows()
return False
def reposition(self):
pass
def main(argv):
global joint_detections
os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
""" Initial tracker
"""
tracker = tracking_module.SelfTracker([FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
""" Build network graph
"""
model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
saver = tf.train.Saver()
""" Get output node
"""
output_node = tf.get_default_graph().get_tensor_by_name(name=FLAGS.output_node_names)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
model_path_suffix = os.path.join(FLAGS.network_def,
'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
'joints_{}'.format(FLAGS.num_of_joints),
'stages_{}'.format(FLAGS.cpm_stages),
'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr, FLAGS.lr_decay_rate,
FLAGS.lr_decay_step)
)
model_save_dir = os.path.join('models',
'weights',
model_path_suffix)
print('Load model from [{}]'.format(os.path.join(model_save_dir, FLAGS.model_path)))
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, 'models/weights/cpm_hand')
# Check weights
for variable in tf.global_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create webcam instance
if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
cam = cv2.VideoCapture(FLAGS.cam_id)
# Create kalman filters
if FLAGS.use_kalman:
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
output_node,
],
feed_dict={model.input_images: test_img_input}
)
print('fps: %.2f' % (1 / (time.time() - t1)))
correct_and_draw_hand(test_img,
cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)),
kalman_filter_array, tracker, tracker.input_crop_ratio, test_img)
# Show visualized image
# demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', test_img.astype(np.uint8))
cv2.waitKey(0)
elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']:
while True:
# # Prepare input image
_, full_img = cam.read()
test_img = tracker.tracking_by_joints(full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale,
test_img_copy)
cv2.imshow('local_img', local_img.astype(np.uint8))
cv2.imshow('global_img', full_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
elif FLAGS.DEMO_TYPE == 'Joint_HM':
while True:
# Prepare input image
test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :,
0:FLAGS.num_of_joints].reshape(
(FLAGS.heatmap_size, FLAGS.heatmap_size, FLAGS.num_of_joints))
demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.num_of_joints, 2))
for joint_num in range(FLAGS.num_of_joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
else:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot FLAGS.limbs
for limb_num in range(len(FLAGS.limbs)):
if np.min(demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]]) > -2000 and np.min(
demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0]
y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1]
x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0]
y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1]
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
color_code_num = limb_num // 4
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.fillConvexPoly(test_img, polygon, color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)),
'constant', constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def eye(
timebase,
is_alive_flag,
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
version,
eye_id,
overwrite_cap_settings=None,
):
"""reads eye video and detects the pupil.
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates.
Reacts to notifications:
``set_detection_mapping_mode``: Sets detection method
``eye_process.should_stop``: Stops the eye process
``recording.started``: Starts recording eye video
``recording.stopped``: Stops recording eye video
``frame_publishing.started``: Starts frame publishing
``frame_publishing.stopped``: Stops frame publishing
Emits notifications:
``eye_process.started``: Eye process started
``eye_process.stopped``: Eye process stopped
Emits data:
``pupil.<eye id>``: Pupil data for eye with id ``<eye id>``
``frame.eye.<eye id>``: Eye frames with id ``<eye id>``
"""
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
ipc_socket = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
pupil_socket = zmq_tools.Msg_Streamer(zmq_ctx, ipc_pub_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=("notify",))
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
if is_alive_flag.value:
# indicates eye process that this is a duplicated startup
logger.warning("Aborting redundant eye process startup")
return
with Is_Alive_Manager(is_alive_flag, ipc_socket, eye_id, logger):
# general imports
import traceback
import numpy as np
import cv2
# display
import glfw
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline
from pyglui.cygl.utils import Named_Texture
from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen
from gl_utils import make_coord_system_pixel_based
from gl_utils import make_coord_system_norm_based
from gl_utils import is_window_visible, glViewport
from ui_roi import UIRoi
# monitoring
import psutil
# helpers/utils
from uvc import get_time_monotonic
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, timer
from av_writer import JPEG_Writer, AV_Writer
from ndsi import H264Writer
from video_capture import source_classes
from video_capture import manager_classes
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D, Detector_Dummy
pupil_detectors = {
Detector_2D.__name__: Detector_2D,
Detector_3D.__name__: Detector_3D,
Detector_Dummy.__name__: Detector_Dummy,
}
# UI Platform tweaks
if platform.system() == "Linux":
scroll_factor = 10.0
window_position_default = (600, 300 * eye_id + 30)
elif platform.system() == "Windows":
scroll_factor = 10.0
window_position_default = (600, 90 + 300 * eye_id)
else:
scroll_factor = 1.0
window_position_default = (600, 300 * eye_id)
icon_bar_width = 50
window_size = None
camera_render_size = None
hdpi_factor = 1.0
# g_pool holds variables for this process
g_pool = SimpleNamespace()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = "capture"
g_pool.process = "eye{}".format(eye_id)
g_pool.timebase = timebase
g_pool.ipc_pub = ipc_socket
def get_timestamp():
return get_time_monotonic() - g_pool.timebase.value
g_pool.get_timestamp = get_timestamp
g_pool.get_now = get_time_monotonic
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
nonlocal hdpi_factor
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_iconify(window, iconified):
g_pool.iconified = iconified
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x *= hdpi_factor
y *= hdpi_factor
g_pool.gui.update_mouse(x, y)
if g_pool.u_r.active_edit_pt:
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
pos = denormalize(pos, g_pool.capture.frame_size)
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx, pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
plugins = (g_pool.capture_manager, g_pool.capture)
# call `on_drop` callbacks until a plugin indicates
# that it has consumed the event (by returning True)
any(p.on_drop(paths) for p in plugins)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, "user_settings_eye{}".format(eye_id))
)
if VersionFormat(session_settings.get("version", "0.0")) != g_pool.version:
logger.info(
"Session setting are from a different version of this app. I will not use those."
)
session_settings.clear()
g_pool.iconified = False
g_pool.capture = None
g_pool.capture_manager = None
g_pool.flip = session_settings.get("flip", False)
g_pool.display_mode = session_settings.get("display_mode", "camera_image")
g_pool.display_mode_info_text = {
"camera_image": "Raw eye camera image. This uses the least amount of CPU power",
"roi": "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
"algorithm": "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below.",
}
capture_manager_settings = session_settings.get(
"capture_manager_settings", ("UVC_Manager", {})
)
manager_class_name, manager_settings = capture_manager_settings
manager_class_by_name = {c.__name__: c for c in manager_classes}
g_pool.capture_manager = manager_class_by_name[manager_class_name](
g_pool, **manager_settings
)
if eye_id == 0:
cap_src = ["Pupil Cam3 ID0", "Pupil Cam2 ID0", "Pupil Cam1 ID0", "HD-6000"]
else:
cap_src = ["Pupil Cam3 ID1", "Pupil Cam2 ID1", "Pupil Cam1 ID1"]
# Initialize capture
default_settings = (
"UVC_Source",
{"preferred_names": cap_src, "frame_size": (320, 240), "frame_rate": 120},
)
capture_source_settings = overwrite_cap_settings or session_settings.get(
"capture_settings", default_settings
)
source_class_name, source_settings = capture_source_settings
source_class_by_name = {c.__name__: c for c in source_classes}
g_pool.capture = source_class_by_name[source_class_name](
g_pool, **source_settings
)
assert g_pool.capture
g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0]))
roi_user_settings = session_settings.get("roi")
if roi_user_settings and tuple(roi_user_settings[-1]) == g_pool.u_r.get()[-1]:
g_pool.u_r.set(roi_user_settings)
pupil_detector_settings = session_settings.get("pupil_detector_settings", None)
last_pupil_detector = pupil_detectors[
session_settings.get("last_pupil_detector", Detector_2D.__name__)
]
g_pool.pupil_detector = last_pupil_detector(g_pool, pupil_detector_settings)
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.deinit_ui()
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_ui()
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
# Initialize glfw
glfw.glfwInit()
title = "Pupil Capture - eye {}".format(eye_id)
width, height = g_pool.capture.frame_size
width *= 2
height *= 2
width += icon_bar_width
width, height = session_settings.get("window_size", (width, height))
main_window = glfw.glfwCreateWindow(width, height, title, None, None)
window_pos = session_settings.get("window_position", window_position_default)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# UI callback functions
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_ndarray(np.ones((1, 1), dtype=np.uint8) + 125)
# setup GUI
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
g_pool.menubar = ui.Scrolling_Menu(
"Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left"
)
g_pool.iconbar = ui.Scrolling_Menu(
"Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos="hidden"
)
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.iconbar)
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(
ui.Selector(
"gui_user_scale",
g_pool,
setter=set_scale,
selection=[0.8, 0.9, 1.0, 1.1, 1.2],
label="Interface Size",
)
)
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width *= 2
f_height *= 2
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
def uroi_on_mouse_button(button, action, mods):
if g_pool.display_mode == "roi":
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
# if the roi interacts we dont want
# the gui to interact as well
return
elif action == glfw.GLFW_PRESS:
x, y = glfw.glfwGetCursorPos(main_window)
# pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
x *= hdpi_factor
y *= hdpi_factor
pos = normalize((x, y), camera_render_size)
if g_pool.flip:
pos = 1 - pos[0], 1 - pos[1]
# Position in img pixels
pos = denormalize(
pos, g_pool.capture.frame_size
) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(
pos, g_pool.u_r.handle_size, g_pool.u_r.handle_size
):
# if the roi interacts we dont want
# the gui to interact as well
return
general_settings.append(ui.Button("Reset window size", set_window_size))
general_settings.append(ui.Switch("flip", g_pool, label="Flip image display"))
general_settings.append(
ui.Selector(
"display_mode",
g_pool,
setter=set_display_mode_info,
selection=["camera_image", "roi", "algorithm"],
labels=["Camera Image", "ROI", "Algorithm"],
label="Mode",
)
)
g_pool.display_mode_info = ui.Info_Text(
g_pool.display_mode_info_text[g_pool.display_mode]
)
general_settings.append(g_pool.display_mode_info)
detector_selector = ui.Selector(
"pupil_detector",
getter=lambda: g_pool.pupil_detector.__class__,
setter=set_detector,
selection=[Detector_Dummy, Detector_2D, Detector_3D],
labels=["disabled", "C++ 2d detector", "C++ 3d detector"],
label="Detection method",
)
general_settings.append(detector_selector)
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
toggle_general_settings(False)
g_pool.pupil_detector.init_ui()
g_pool.capture.init_ui()
g_pool.capture_manager.init_ui()
g_pool.writer = None
def replace_source(source_class_name, source_settings):
g_pool.capture.deinit_ui()
g_pool.capture.cleanup()
g_pool.capture = source_class_by_name[source_class_name](
g_pool, **source_settings
)
g_pool.capture.init_ui()
if g_pool.writer:
logger.info("Done recording.")
try:
g_pool.writer.release()
except RuntimeError:
logger.error("No eye video recorded")
g_pool.writer = None
g_pool.replace_source = replace_source # for ndsi capture
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetWindowIconifyCallback(main_window, on_iconify)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
# load last gui configuration
g_pool.gui.configuration = session_settings.get("ui_config", {})
# set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = g_pool.get_timestamp()
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 50)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = "CPU %0.1f"
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 50)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
g_pool.graphs = [cpu_graph, fps_graph]
# set the last saved window size
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
should_publish_frames = False
frame_publish_format = "jpeg"
frame_publish_format_recent_warning = False
# create a timer to control window update frequency
window_update_timer = timer(1 / 60)
def window_should_update():
return next(window_update_timer)
logger.warning("Process started.")
frame = None
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if notify_sub.new_data:
t, notification = notify_sub.recv()
subject = notification["subject"]
if subject.startswith("eye_process.should_stop"):
if notification["eye_id"] == eye_id:
break
elif subject == "set_detection_mapping_mode":
if notification["mode"] == "3d":
if not isinstance(g_pool.pupil_detector, Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
elif notification["mode"] == "2d":
if not isinstance(g_pool.pupil_detector, Detector_2D):
set_detector(Detector_2D)
detector_selector.read_only = False
else:
if not isinstance(g_pool.pupil_detector, Detector_Dummy):
set_detector(Detector_Dummy)
detector_selector.read_only = True
elif subject == "recording.started":
if notification["record_eye"] and g_pool.capture.online:
record_path = notification["rec_path"]
raw_mode = notification["compression"]
logger.info("Will save eye video to: {}".format(record_path))
video_path = os.path.join(
record_path, "eye{}.mp4".format(eye_id)
)
if raw_mode and frame and g_pool.capture.jpeg_support:
g_pool.writer = JPEG_Writer(
video_path, g_pool.capture.frame_rate
)
elif hasattr(g_pool.capture._recent_frame, "h264_buffer"):
g_pool.writer = H264Writer(
video_path,
g_pool.capture.frame_size[0],
g_pool.capture.frame_size[1],
g_pool.capture.frame_rate,
)
else:
g_pool.writer = AV_Writer(
video_path, g_pool.capture.frame_rate
)
elif subject == "recording.stopped":
if g_pool.writer:
logger.info("Done recording.")
try:
g_pool.writer.release()
except RuntimeError:
logger.error("No eye video recorded")
g_pool.writer = None
elif subject.startswith("meta.should_doc"):
ipc_socket.notify(
{
"subject": "meta.doc",
"actor": "eye{}".format(eye_id),
"doc": eye.__doc__,
}
)
elif subject.startswith("frame_publishing.started"):
should_publish_frames = True
frame_publish_format = notification.get("format", "jpeg")
elif subject.startswith("frame_publishing.stopped"):
should_publish_frames = False
frame_publish_format = "jpeg"
elif (
subject.startswith("start_eye_capture")
and notification["target"] == g_pool.process
):
replace_source(notification["name"], notification["args"])
elif notification["subject"].startswith("pupil_detector.set_property"):
target_process = notification.get("target", g_pool.process)
should_apply = target_process == g_pool.process
if should_apply:
try:
property_name = notification["name"]
property_value = notification["value"]
if "2d" in notification["subject"]:
g_pool.pupil_detector.set_2d_detector_property(
property_name, property_value
)
elif "3d" in notification["subject"]:
if not isinstance(g_pool.pupil_detector, Detector_3D):
raise ValueError(
"3d properties are only available"
" if 3d detector is active"
)
g_pool.pupil_detector.set_3d_detector_property(
property_name, property_value
)
else:
raise KeyError(
"Notification subject does not "
"specifiy detector type."
)
logger.debug(
"`{}` property set to {}".format(
property_name, property_value
)
)
except KeyError:
logger.error("Malformed notification received")
logger.debug(traceback.format_exc())
except (ValueError, TypeError):
logger.error("Invalid property or value")
logger.debug(traceback.format_exc())
elif notification["subject"].startswith(
"pupil_detector.broadcast_properties"
):
target_process = notification.get("target", g_pool.process)
should_respond = target_process == g_pool.process
if should_respond:
props = g_pool.pupil_detector.get_detector_properties()
properties_broadcast = {
"subject": "pupil_detector.properties.{}".format(eye_id),
**props, # add properties to broadcast
}
ipc_socket.notify(properties_broadcast)
g_pool.capture.on_notify(notification)
g_pool.capture_manager.on_notify(notification)
# Get an image from the grabber
event = {}
g_pool.capture.recent_events(event)
frame = event.get("frame")
g_pool.capture_manager.recent_events(event)
if frame:
f_width, f_height = g_pool.capture.frame_size
if (g_pool.u_r.array_shape[0], g_pool.u_r.array_shape[1]) != (
f_height,
f_width,
):
g_pool.pupil_detector.on_resolution_change(
(g_pool.u_r.array_shape[1], g_pool.u_r.array_shape[0]),
g_pool.capture.frame_size,
)
g_pool.u_r = UIRoi((f_height, f_width))
if should_publish_frames:
try:
if frame_publish_format == "jpeg":
data = frame.jpeg_buffer
elif frame_publish_format == "yuv":
data = frame.yuv_buffer
elif frame_publish_format == "bgr":
data = frame.bgr
elif frame_publish_format == "gray":
data = frame.gray
assert data is not None
except (AttributeError, AssertionError, NameError):
if not frame_publish_format_recent_warning:
frame_publish_format_recent_warning = True
logger.warning(
'{}s are not compatible with format "{}"'.format(
type(frame), frame_publish_format
)
)
else:
frame_publish_format_recent_warning = False
pupil_socket.send(
{
"topic": "frame.eye.{}".format(eye_id),
"width": frame.width,
"height": frame.height,
"index": frame.index,
"timestamp": frame.timestamp,
"format": frame_publish_format,
"__raw_data__": [data],
}
)
t = frame.timestamp
dt, ts = t - ts, t
try:
fps_graph.add(1.0 / dt)
except ZeroDivisionError:
pass
if g_pool.writer:
g_pool.writer.write_video_frame(frame)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(
frame, g_pool.u_r, g_pool.display_mode == "algorithm"
)
if result is not None:
result["id"] = eye_id
result["topic"] = "pupil.{}".format(eye_id)
pupil_socket.send(result)
cpu_graph.update()
# GL drawing
if window_should_update():
if is_window_visible(main_window):
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
if frame:
# switch to work in normalized coordinate space
if g_pool.display_mode == "algorithm":
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ("camera_image", "roi"):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
glViewport(0, 0, *camera_render_size)
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
f_width, f_height = g_pool.capture.frame_size
make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
if frame and result:
if result["method"] == "3d c++":
eye_ball = result["projected_sphere"]
try:
pts = cv2.ellipse2Poly(
(
int(eye_ball["center"][0]),
int(eye_ball["center"][1]),
),
(
int(eye_ball["axes"][0] / 2),
int(eye_ball["axes"][1] / 2),
),
int(eye_ball["angle"]),
0,
360,
8,
)
except ValueError as e:
pass
else:
draw_polyline(
pts,
2,
RGBA(0.0, 0.9, 0.1, result["model_confidence"]),
)
if result["confidence"] > 0:
if "ellipse" in result:
pts = cv2.ellipse2Poly(
(
int(result["ellipse"]["center"][0]),
int(result["ellipse"]["center"][1]),
),
(
int(result["ellipse"]["axes"][0] / 2),
int(result["ellipse"]["axes"][1] / 2),
),
int(result["ellipse"]["angle"]),
0,
360,
15,
)
confidence = result["confidence"] * 0.7
draw_polyline(pts, 1, RGBA(1.0, 0, 0, confidence))
draw_points(
[result["ellipse"]["center"]],
size=20,
color=RGBA(1.0, 0.0, 0.0, confidence),
sharpness=1.0,
)
glViewport(0, 0, *camera_render_size)
make_coord_system_pixel_based((f_height, f_width, 3), g_pool.flip)
# render the ROI
g_pool.u_r.draw(g_pool.gui.scale)
if g_pool.display_mode == "roi":
g_pool.u_r.draw_points(g_pool.gui.scale)
glViewport(0, 0, *window_size)
make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)
# render graphs
fps_graph.draw()
cpu_graph.draw()
# render GUI
unused_elements = g_pool.gui.update()
for butt in unused_elements.buttons:
uroi_on_mouse_button(*butt)
make_coord_system_pixel_based((*window_size[::-1], 3), g_pool.flip)
g_pool.pupil_detector.visualize() # detector decides if we visualize or not
# update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
# END while running
# in case eye recording was still runnnig: Save&close
if g_pool.writer:
logger.info("Done recording eye.")
g_pool.writer = None
glfw.glfwRestoreWindow(main_window) # need to do this for windows os
# save session persistent settings
session_settings["gui_scale"] = g_pool.gui_user_scale
session_settings["roi"] = g_pool.u_r.get()
session_settings["flip"] = g_pool.flip
session_settings["display_mode"] = g_pool.display_mode
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["capture_settings"] = (
g_pool.capture.class_name,
g_pool.capture.get_init_dict(),
)
session_settings["capture_manager_settings"] = (
g_pool.capture_manager.class_name,
g_pool.capture_manager.get_init_dict(),
)
session_settings["window_position"] = glfw.glfwGetWindowPos(main_window)
session_settings["version"] = str(g_pool.version)
session_settings[
"last_pupil_detector"
] = g_pool.pupil_detector.__class__.__name__
session_settings[
"pupil_detector_settings"
] = g_pool.pupil_detector.get_settings()
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings["window_size"] = session_window_size
session_settings.close()
g_pool.capture.deinit_ui()
g_pool.capture_manager.deinit_ui()
g_pool.pupil_detector.deinit_ui()
g_pool.pupil_detector.cleanup()
g_pool.capture_manager.cleanup()
g_pool.capture.cleanup()
glfw.glfwDestroyWindow(main_window)
g_pool.gui.terminate()
glfw.glfwTerminate()
logger.info("Process shutting down.")
def gl_display(self):
if self.fixation is not None:
abs_fixation = denormalize(self.fixation,self.g_pool.capture.frame_size, flip_y=True)
ellipse = cv2.ellipse2Poly((int(abs_fixation[0]), int(abs_fixation[1])),(25, 25),0,0,360,15)
draw_gl_polyline(ellipse,(0.,0.,.5,.75),'Polygon')
def main():
param, model = config_reader()
model_xml = "/home/yue/Realtime_Multi-Person_Pose_Estimation/model/_trained_COCO/pose_iter_440000.xml"
model_bin = os.path.splitext(model_xml)[0] + ".bin"
prob_threshold = 0.5
labels_map = None
print("Initializing plugin for CPU device...")
plugin = IEPlugin(device="CPU", plugin_dirs=None)
print("Adding CPU extenstions...")
plugin.add_cpu_extension("/opt/intel/computer_vision_sdk/deployment_tools/inference_engine/lib/ubuntu_16.04/intel64/libcpu_extension_sse4.so")
print("Reading IR...")
net = IENetwork.from_ir(model=model_xml, weights=model_bin)
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
print("Loading IR to the plugin...")
exec_net = plugin.load(network=net, num_requests=2)
# Read and pre-process input image
n, c, h, w = net.inputs[input_blob]
print((n,c,h,w))
input_stream = "../sample_video/dance.mp4"
cap = cv.VideoCapture(input_stream)
cur_request_id = 0
next_request_id = 1
if (cap.isOpened()== False):
print("Error opening video stream or file")
print("Starting inference in async mode...")
is_async_mode = True
render_time = 0
# find connection in the specified sequence, center 29 is in the position 15
limbSeq = [[2,3], [2,6], [3,4], [4,5], [6,7], [7,8], [2,9], [9,10], \
[10,11], [2,12], [12,13], [13,14], [2,1], [1,15], [15,17], \
[1,16], [16,18], [3,17], [6,18]]
# the middle joints heatmap correpondence
mapIdx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], \
[23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], \
[55,56], [37,38], [45,46]]
colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
[0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
[170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
fps = 0
while cap.isOpened():
ret, frame = cap.read()
if not ret:
break
initial_w = cap.get(3)
initial_h = cap.get(4)
image = cv.resize(frame, (w, h))
image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW
image = image.reshape((1,3,368,368))/256 - 0.5
img_show = frame;
# Main sync point:
# in the truly Async mode we start the NEXT infer request, while waiting for the CURRENT to complete
# in the regular mode we start the CURRENT request and immediately wait for it's completion
inf_start = time.time()
res = exec_net.infer(inputs={input_blob: image})
inf_end = time.time()
det_time = inf_end - inf_start
# Parse detection results of the current request
res_heatMap = res['Mconv7_stage6_L2']
res_paf = res['Mconv7_stage6_L1']
#Process outputs
res_heatMap = np.squeeze(res_heatMap, axis=0)
res_paf = np.squeeze(res_paf, axis=0)
# extract outputs, resize, and remove padding
heatmap = np.transpose(res_heatMap, (1,2,0)) # output 1 is heatmaps
heatmap = cv.resize(heatmap, (frame.shape[1], frame.shape[0]), interpolation=cv.INTER_CUBIC)
paf = np.transpose(res_paf, (1,2,0)) # output 0 is PAFs
paf = cv.resize(paf, (frame.shape[1], frame.shape[0]), interpolation=cv.INTER_CUBIC)
heatmap_avg = heatmap
paf_avg = paf
U = paf_avg[:,:,16] * -1
V = paf_avg[:,:,17]
X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0]))
M = np.zeros(U.shape, dtype='bool')
M[U**2 + V**2 < 0.5 * 0.5] = True
U = ma.masked_array(U, mask=M)
V = ma.masked_array(V, mask=M)
all_peaks = []
peak_counter = 0
for part in range(19-1):
x_list = []
y_list = []
map_ori = heatmap_avg[:,:,part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:,:] = map[:-1,:]
map_right = np.zeros(map.shape)
map_right[:-1,:] = map[1:,:]
map_up = np.zeros(map.shape)
map_up[:,1:] = map[:,:-1]
map_down = np.zeros(map.shape)
map_down[:,:-1] = map[:,1:]
peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param['thre1']))
peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf_avg[:,:,[x-19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0]-1]
candB = all_peaks[limbSeq[k][1]-1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if(nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
norm = math.sqrt(vec[0]*vec[0] + vec[1]*vec[1])
vec = np.divide(vec, norm)
startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num)))
vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
score_with_dist_prior = sum(score_midpts)/len(score_midpts) + min(0.5*frame.shape[0]/norm-1, 0)
criterion1 = len(np.nonzero(score_midpts > param['thre2'])[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior+candA[i][2]+candB[j][2]])
connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
connection = np.zeros((0,5))
for c in range(len(connection_candidate)):
i,j,s = connection_candidate[c][0:3]
if(i not in connection[:,3] and j not in connection[:,4]):
connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
if(len(connection) >= min(nA, nB)):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:,0]
partBs = connection_all[k][:,1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])): #= 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): #1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if(subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
print ("found = 2")
membership = ((subset[j1]>=0).astype(int) + (subset[j2]>=0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: #merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(candidate[connection_all[k][i,:2].astype(int), 2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
deleteIdx = [];
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2]/subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
stickwidth = 2
cmap = matplotlib.cm.get_cmap('hsv')
for i in range(18):
rgba = np.array(cmap(1 - i/18. - 1./36))
rgba[0:3] *= 255
for j in range(len(all_peaks[i])):
cv.circle(img_show, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
to_plot = cv.addWeighted(frame, 0.3, img_show, 0.7, 0)
for i in range(17):
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[i])-1]
if -1 in index:
continue
cur_canvas = img_show.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
cv.fillConvexPoly(cur_canvas, polygon, colors[i])
img_show = cv.addWeighted(img_show, 0.4, cur_canvas, 0.6, 0)
#################################################################################################################
# Draw performance stats
inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \
"Inference time: {:.3f} ms".format(det_time * 1000)
render_time_message = "OpenCV rendering time: {:.3f} ms".format(render_time * 1000)
async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \
"Async mode is off. Processing request {}".format(cur_request_id)
fps_message = "FPS: {:.1f}".format(fps);
cv.putText(img_show, inf_time_message, (15, 15), cv.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
cv.putText(img_show, render_time_message, (15, 30), cv.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
cv.putText(img_show, fps_message, (15, 45), cv.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
render_start = time.time()
cv.imshow("Detection Results", img_show)
render_end = time.time()
render_time = render_end - render_start
fps = float(60/(render_time*1000))
key = cv.waitKey(1)
if key == 27:
break
if (9 == key):
is_async_mode = not is_async_mode
print("Switched to {} mode".format("async" if is_async_mode else "sync"))
if is_async_mode:
cur_request_id, next_request_id = next_request_id, cur_request_id
cv.destroyAllWindows()
del exec_net
del plugin
def process(model, input_path):
origin_img = cv2.imread(input_path)
normed_img = normalize(origin_img)
height, width, _ = normed_img.shape
multiplier = [x * boxsize / height for x in scale_search]
heatmap_avg = np.zeros((height, width, 19)) # num_point
paf_avg = np.zeros((height, width, 38)) # num_vector
for m in range(len(multiplier)):
scale = multiplier[m]
# preprocess
imgToTest = cv2.resize(normed_img, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
imgToTest_padded, pad = padRightDownCorner(imgToTest, stride, padValue)
input_img = np.transpose(imgToTest_padded[:, :, :, np.newaxis],
(3, 2, 0, 1)) # required shape (1, c, h, w)
mask = np.ones(
(1, 1, input_img.shape[2] / stride, input_img.shape[3] / stride),
dtype=np.float32)
input_var = torch.autograd.Variable(torch.from_numpy(input_img).cuda())
mask_var = torch.autograd.Variable(torch.from_numpy(mask).cuda())
# get the features
vec1, heat1, vec2, heat2, vec3, heat3, vec4, heat4, vec5, heat5, vec6, heat6 = model(
input_var, mask_var)
# get the heatmap
heatmap = heat6.data.cpu().numpy()
heatmap = np.transpose(np.squeeze(heatmap), (1, 2, 0)) # (h, w, c)
heatmap = cv2.resize(heatmap, (0, 0),
fx=stride,
fy=stride,
interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imgToTest_padded.shape[0] -
pad[2], :imgToTest_padded.shape[1] - pad[3], :]
heatmap = cv2.resize(heatmap, (width, height),
interpolation=cv2.INTER_CUBIC)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
# get the paf
paf = vec6.data.cpu().numpy()
paf = np.transpose(np.squeeze(paf), (1, 2, 0)) # (h, w, c)
paf = cv2.resize(paf, (0, 0),
fx=stride,
fy=stride,
interpolation=cv2.INTER_CUBIC)
paf = paf[:imgToTest_padded.shape[0] -
pad[2], :imgToTest_padded.shape[1] - pad[3], :]
paf = cv2.resize(paf, (width, height), interpolation=cv2.INTER_CUBIC)
paf_avg = paf_avg + paf / len(multiplier)
all_peaks = [] # all of the possible points by classes.
peak_counter = 0
for part in range(1, 19):
map_ori = heatmap_avg[:, :, part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[:, 1:] = map[:, :-1]
map_right = np.zeros(map.shape)
map_right[:, :-1] = map[:, 1:]
map_up = np.zeros(map.shape)
map_up[1:, :] = map[:-1, :]
map_down = np.zeros(map.shape)
map_down[:-1, :] = map[1:, :]
# get the salient point and its score > thre_point
peaks_binary = np.logical_and.reduce(
(map >= map_left, map >= map_right, map >= map_up, map >= map_down,
map > thre_point))
peaks = list(
zip(np.nonzero(peaks_binary)[1],
np.nonzero(peaks_binary)[0])) # (w, h)
# a point format: (w, h, score, number)
peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [
peaks_with_score[i] + (id[i], ) for i in range(len(id))
]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = [] # save all of the possible lines by classes.
special_k = [] # save the lines, which haven't legal points.
mid_num = 10 # could adjust to accelerate (small) or improve accuracy(large).
for k in range(len(mapIdx)):
score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0] - 1]
candB = all_peaks[limbSeq[k][1] - 1]
lenA = len(candA)
lenB = len(candB)
if lenA != 0 and lenB != 0:
connection_candidate = []
for i in range(lenA):
for j in range(lenB):
vec = np.subtract(candB[j][:2],
candA[i][:2]) # the vector of BA
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
if norm == 0:
continue
vec = np.divide(vec, norm)
startend = list(
zip(np.linspace(candA[i][0], candB[j][0], num=mid_num),
np.linspace(candA[i][1], candB[j][1],
num=mid_num)))
# get the vector between A and B.
vec_x = np.array([
score_mid[int(round(startend[I][1])),
int(round(startend[I][0])), 0]
for I in range(len(startend))
])
vec_y = np.array([
score_mid[int(round(startend[I][1])),
int(round(startend[I][0])), 1]
for I in range(len(startend))
])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(
vec_y, vec[1])
score_with_dist_prior = sum(score_midpts) / len(
score_midpts) + min(0.5 * height / norm - 1, 0) # ???
criterion1 = len(np.nonzero(
score_midpts > thre_line)[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([
i, j, score_with_dist_prior,
score_with_dist_prior + candA[i][2], candB[j][2]
])
# sort the possible line from large to small order.
connection_candidate = sorted(connection_candidate,
key=lambda x: x[2],
reverse=True)
connection = np.zeros((0, 5))
for c in range(len(connection_candidate)):
i, j, s = connection_candidate[c][0:3]
if (i not in connection[:, 3] and j not in connection[:, 4]):
# the number of A point, the number of B point, score, A point, B point
connection = np.vstack(
[connection, [candA[i][3], candB[j][3], s, i, j]])
if len(connection) >= min(lenA, lenB):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:, 0]
partBs = connection_all[k][:, 1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])):
found = 0
flag = [False, False]
subset_idx = [-1, -1]
for j in range(len(subset)):
# fix the bug, found == 2 and not joint will lead someone occur more than once.
# if more than one, we choose the subset, which has a higher score.
if subset[j][indexA] == partAs[i]:
if flag[0] == False:
flag[0] = found
subset_idx[found] = j
flag[0] = True
found += 1
else:
ids = subset_idx[flag[0]]
if subset[ids][-1] < subset[j][-1]:
subset_idx[flag[0]] = j
if subset[j][indexB] == partBs[i]:
if flag[1] == False:
flag[1] = found
subset_idx[found] = j
flag[1] = True
found += 1
else:
ids = subset_idx[flag[1]]
if subset[ids][-1] < subset[j][-1]:
subset_idx[flag[1]] = j
if found == 1:
j = subset_idx[0]
if (subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int),
2] + connection_all[k][i][2]
elif found == 2: # if found equals to 2 and disjoint, merge them
j1, j2 = subset_idx
membership = ((subset[j1] >= 0).astype(int) +
(subset[j2] >= 0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: # merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[
partBs[i].astype(int), 2] + connection_all[k][i][2]
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(
candidate[connection_all[k][i, :2].astype(int),
2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete som rows of subset which has few parts occur
deleteIdx = []
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
# draw points
canvas = cv2.imread(input_path)
for i in range(18):
for j in range(len(all_peaks[i])):
cv2.circle(canvas,
all_peaks[i][j][0:2],
4,
colors[i],
thickness=-1)
# draw lines
for i in range(17):
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[i]) - 1]
if -1 in index:
continue
cur_canvas = canvas.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly(
(int(mY), int(mX)), (int(length / 2), stickwidth), int(angle),
0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
return canvas
def main(argv):
tf_device = '/gpu:0'
with tf.device(tf_device):
"""Build graph
"""
if FLAGS.color_channel == 'RGB':
input_data = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
else:
input_data = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='input_image')
center_map = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size,
FLAGS.input_size,
FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map,
[1, FLAGS.input_size, FLAGS.input_size, 1])
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create kalman filters
if FLAGS.KALMAN_ON:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
# read in video / flow frames
if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
# OpenCV can only read in '.avi' files
cam = imageio.get_reader(FLAGS.DEMO_TYPE)
else:
cam = cv2.VideoCapture(FLAGS.cam_num)
# iamge processing
with tf.device(tf_device):
if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
ori_fps = cam.get_meta_data()['fps']
print('This video fps is %f' % ori_fps)
video_length = cam.get_length()
writer_path = os.path.join('results',
os.path.basename(FLAGS.DEMO_TYPE))
# !! OpenCV can only write in .avi
cv_writer = cv2.VideoWriter(
writer_path + '.avi',
# cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'),
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
ori_fps,
(FLAGS.input_size, FLAGS.input_size))
# imageio_writer = imageio.get_writer(writer_path, fps=ori_fps)
try:
for it, im in enumerate(cam):
test_img_t = time.time()
test_img = cpm_utils.read_image(im, [], FLAGS.input_size,
'VIDEO')
test_img_resize = cv2.resize(
test_img, (FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - test_img_t))
if FLAGS.color_channel == 'GRAY':
test_img_resize = mgray(test_img_resize, test_img)
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={
'input_image:0': test_img_input,
'center_map:0': test_center_map
})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS,
stage_heatmap_np,
kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if (cv2.waitKey(1) == ord('q')): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
cv_writer.write(demo_img.astype(np.uint8))
# imageio_writer.append_data(demo_img[:, :, 1])
except KeyboardInterrupt:
print('Stopped! {}/{} frames captured!'.format(
it, video_length))
finally:
cv_writer.release()
# imageio_writer.close()
else:
while True:
test_img_t = time.time()
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [],
FLAGS.input_size, 'IMAGE')
else:
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
test_img_resize = cv2.resize(
test_img, (FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - test_img_t))
if FLAGS.color_channel == 'GRAY':
test_img_resize = mgray(test_img_resize, test_img)
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={
'input_image:0': test_img_input,
'center_map:0': test_center_map
})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS,
stage_heatmap_np,
kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(0) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'MULTI':
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={
'input_image:0': test_img_input,
'center_map:0': test_center_map
})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS,
stage_heatmap_np,
kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'SINGLE':
# Inference
fps_t = time.time()
stage_heatmap_np = sess.run(
[model.stage_heatmap[5]],
feed_dict={
'input_image:0': test_img_input,
'center_map:0': test_center_map
})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS,
stage_heatmap_np,
kalman_filter_array)
cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'HM':
# Inference
fps_t = time.time()
stage_heatmap_np = sess.run(
[model.stage_heatmap[FLAGS.stages - 1]],
feed_dict={
'input_image:0': test_img_input,
'center_map:0': test_center_map
})
print('fps: %.2f' % (1 / (time.time() - fps_t)))
# demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape(
# (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = stage_heatmap_np[-1][
0, :, :, 0:FLAGS.joints].reshape(
(FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = cv2.resize(
demo_stage_heatmap,
(FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.joints, 2))
for joint_num in range(FLAGS.joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (
255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(
np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num]))
else:
joint_color = map(
lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num])
cv2.circle(test_img,
center=(joint_coord[1],
joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
else:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num]))
else:
joint_color = map(
lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num])
cv2.circle(test_img,
center=(joint_coord[1],
joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :,
joint_num].astype(np.uint8)
tmp = cv2.putText(
tmp,
'Min:' +
str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp = cv2.putText(
tmp,
'Mean:' +
str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot limbs
for limb_num in range(len(limbs)):
if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]
) > -2000 and np.min(
demo_stage_heatmap[:, :,
limbs[limb_num][1]]
) > -2000:
x1 = joint_coord_set[limbs[limb_num][0], 0]
y1 = joint_coord_set[limbs[limb_num][0], 1]
x2 = joint_coord_set[limbs[limb_num][1], 0]
y2 = joint_coord_set[limbs[limb_num][1], 1]
length = ((x1 - x2)**2 + (y1 - y2)**2)**0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(
x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int(
(y1 + y2) / 2), int(
(x1 + x2) / 2)), (int(length / 2), 3),
int(deg), 0, 360, 1)
color_code_num = limb_num // 4
if PYTHON_VERSION == 3:
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4),
joint_color_code[color_code_num]))
else:
limb_color = map(
lambda x: x + 35 * (limb_num % 4),
joint_color_code[color_code_num])
cv2.fillConvexPoly(test_img,
polygon,
color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(
tmp_img,
((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]),
(0, 0)),
'constant',
constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img,
cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300),
cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def vis_frame(im_res, format='coco'):
'''
frame: frame image
im_res: im_res of predictions
format: coco or mpii
return rendered image
'''
if format == 'coco':
l_pair = [
(0, 1), (0, 2), (1, 3), (2, 4), # Head
(5, 6), (5, 7), (7, 9), (6, 8), (8, 10),
(17, 11), (17, 12), # Body
(11, 13), (12, 14), (13, 15), (14, 16)
]
p_color = [(0, 255, 255), (0, 191, 255),(0, 255, 102),(0, 77, 255), (0, 255, 0), #Nose, LEye, REye, LEar, REar
(77,255,255), (77, 255, 204), (77,204,255), (191, 255, 77), (77,191,255), (191, 255, 77), #LShoulder, RShoulder, LElbow, RElbow, LWrist, RWrist
(204,77,255), (77,255,204), (191,77,255), (77,255,191), (127,77,255), (77,255,127), (0, 255, 255)] #LHip, RHip, LKnee, Rknee, LAnkle, RAnkle, Neck
line_color = [(0, 215, 255), (0, 255, 204), (0, 134, 255), (0, 255, 50),
(77,255,222), (77,196,255), (77,135,255), (191,255,77), (77,255,77),
(77,222,255), (255,156,127),
(0,127,255), (255,127,77), (0,77,255), (255,77,36)]
elif format == 'mpii':
l_pair = [
(8, 9), (11, 12), (11, 10), (2, 1), (1, 0),
(13, 14), (14, 15), (3, 4), (4, 5),
(8, 7), (7, 6), (6, 2), (6, 3), (8, 12), (8, 13)
]
p_color = [PURPLE, BLUE, BLUE, RED, RED, BLUE, BLUE, RED, RED, PURPLE, PURPLE, PURPLE, RED, RED, BLUE, BLUE]
line_color = [PURPLE, BLUE, BLUE, RED, RED, BLUE, BLUE, RED, RED, PURPLE, PURPLE, RED, RED, BLUE, BLUE]
else:
raise NotImplementedError
#print(im_res) #added by ashvini
# im_name = im_res['imgname'].split('/')[-1]
# im_name = im_res[2]
#print(frame.shape)
img = np.zeros((360,640,3),np.uint8) #image=frame #added by ashvini
height,width = img.shape[:2]
img = cv2.resize(img,(int(width/2), int(height/2)))
# for human in im_res:
part_line = {}
kp_preds = torch.Tensor(list(zip(im_res[3:20],im_res[20:37])))
#added by ashvini start here
# y = kp_preds[:,0]
# x = kp_preds[:,1]
# y_stand = 90.0
# x_stand = 160.0
# norm_y = max(y)-min(y)
# norm_x = max(x)-min(x)
# y_c = (max(y)+min(y))/2
# x_c = (max(x)+min(x))/2
# #print(norm_y, norm_x)
# kp_preds[:,0] = (((kp_preds[:,0]-y_c)/norm_y)*y_stand)+y_c
# kp_preds[:,1] = (((kp_preds[:,1]-x_c)/norm_x)*x_stand)+x_c
# print(len(kp_preds))
#print(kp_preds)
#print("zero",kp_preds[:,0],type(kp_preds))
#added by ashvini end here
kp_scores =torch.Tensor(im_res[37:54]).unsqueeze(1)
# print(kp_scores.shape)
kp_preds = torch.cat((kp_preds, torch.unsqueeze((kp_preds[5,:]+kp_preds[6,:])/2,0)))
kp_scores = torch.cat((kp_scores, torch.unsqueeze((kp_scores[5,:]+kp_scores[6,:])/2,0)))
# Draw keypoints
for n in range(kp_scores.shape[0]):
if kp_scores[n] <= 0.05:
continue
cor_x, cor_y = int(kp_preds[n, 0]), int(kp_preds[n, 1])
part_line[n] = (int(cor_x/2), int(cor_y/2))
bg = img.copy()
cv2.circle(bg, (int(cor_x/2), int(cor_y/2)), 2, p_color[n], -1)
# Now create a mask of logo and create its inverse mask also
transparency = max(0, min(1, kp_scores[n]))
img = cv2.addWeighted(bg, transparency, img, 1-transparency, 0)
# Draw limbs
for i, (start_p, end_p) in enumerate(l_pair):
if start_p in part_line and end_p in part_line:
start_xy = part_line[start_p]
end_xy = part_line[end_p]
bg = img.copy()
X = (start_xy[0], end_xy[0])
Y = (start_xy[1], end_xy[1])
mX = np.mean(X)
mY = np.mean(Y)
length = ((Y[0] - Y[1]) ** 2 + (X[0] - X[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(Y[0] - Y[1], X[0] - X[1]))
stickwidth = (kp_scores[start_p] + kp_scores[end_p]) + 1
polygon = cv2.ellipse2Poly((int(mX),int(mY)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
cv2.fillConvexPoly(bg, polygon, line_color[i])
#cv2.line(bg, start_xy, end_xy, line_color[i], (2 * (kp_scores[start_p] + kp_scores[end_p])) + 1)
transparency = max(0, min(1, 0.5*(kp_scores[start_p] + kp_scores[end_p])))
img = cv2.addWeighted(bg, transparency, img, 1-transparency, 0)
img = cv2.resize(img,(width,height),interpolation=cv2.INTER_CUBIC)
return img
def handle_one_run1(oriImg, model_, param_, model, mapIdx, limbSeq, colors):
tic = time.time()
# for visualize
canvas = np.copy(oriImg)
multiplier = [
x * model_['boxsize'] / oriImg.shape[0] for x in param_['scale_search']
]
imageToTest = Variable(T.transpose(
T.transpose(T.unsqueeze(torch.from_numpy(oriImg).float(), 0), 2, 3), 1,
2),
volatile=True).cuda()
#print oriImg.shape
scale = model_['boxsize'] / float(oriImg.shape[0])
#print scale
h = int(oriImg.shape[0] * scale)
w = int(oriImg.shape[1] * scale)
pad_h = 0 if (h % model_['stride']
== 0) else model_['stride'] - (h % model_['stride'])
pad_w = 0 if (w % model_['stride']
== 0) else model_['stride'] - (w % model_['stride'])
new_h = h + pad_h
new_w = w + pad_w
imageToTest = cv2.resize(oriImg, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.padRightDownCorner(imageToTest,
model_['stride'],
model_['padValue'])
imageToTest_padded = np.transpose(
np.float32(imageToTest_padded[:, :, :, np.newaxis]),
(3, 2, 0, 1)) / 256 - 0.5
feed = Variable(T.from_numpy(imageToTest_padded)).cuda()
tic = time.time()
out1, out2, out3, out4, out5, out5_1, out5_2, output1, output2, out6 = model(
feed)
heatmap = nn.UpsamplingBilinear2d(
(oriImg.shape[0], oriImg.shape[1])).cuda()(output2)
paf = nn.UpsamplingBilinear2d(
(oriImg.shape[0], oriImg.shape[1])).cuda()(output1)
#print '-------------------------------------'+str(time.time()-tic)
#print heatmap.size()
#print paf.size()
#print type(heatmap)
heatmap_avg = torch.zeros(
(len(multiplier), 19, oriImg.shape[0], oriImg.shape[1])).cuda()
paf_avg = torch.zeros(
(len(multiplier), 38, oriImg.shape[0], oriImg.shape[1])).cuda()
toc = time.time()
#print 'time is %.5f'%(toc-tic)
tic = time.time()
for m in range(len(multiplier)):
scale = multiplier[m]
h = int(oriImg.shape[0] * scale)
w = int(oriImg.shape[1] * scale)
pad_h = 0 if (h % model_['stride']
== 0) else model_['stride'] - (h % model_['stride'])
pad_w = 0 if (w % model_['stride']
== 0) else model_['stride'] - (w % model_['stride'])
new_h = h + pad_h
new_w = w + pad_w
imageToTest = cv2.resize(oriImg, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.padRightDownCorner(
imageToTest, model_['stride'], model_['padValue'])
imageToTest_padded = np.transpose(
np.float32(imageToTest_padded[:, :, :, np.newaxis]),
(3, 2, 0, 1)) / 256 - 0.5
feed = Variable(T.from_numpy(imageToTest_padded)).cuda()
out1, out2, out3, out4, out5, out5_1, out5_2, output1, output2, out6 = model(
feed)
#print output1.size()
#print output2.size()
heatmap = nn.UpsamplingBilinear2d(
(oriImg.shape[0], oriImg.shape[1])).cuda()(output2)
paf = nn.UpsamplingBilinear2d(
(oriImg.shape[0], oriImg.shape[1])).cuda()(output1)
heatmap_avg[m] = heatmap[0].data
paf_avg[m] = paf[0].data
toc = time.time()
#print 'time is %.5f'%(toc-tic)
tic = time.time()
heatmap_avg = T.transpose(
T.transpose(T.squeeze(T.mean(heatmap_avg, 0)), 0, 1), 1, 2).cuda()
paf_avg = T.transpose(T.transpose(T.squeeze(T.mean(paf_avg, 0)), 0, 1), 1,
2).cuda()
heatmap_avg = heatmap_avg.cpu().numpy()
paf_avg = paf_avg.cpu().numpy()
#toc =time.time()
#print 'time is %.5f'%(toc-tic)
#tic = time.time()
all_peaks = []
peak_counter = 0
#maps =
for part in range(18):
map_ori = heatmap_avg[:, :, part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:, :] = map[:-1, :]
map_right = np.zeros(map.shape)
map_right[:-1, :] = map[1:, :]
map_up = np.zeros(map.shape)
map_up[:, 1:] = map[:, :-1]
map_down = np.zeros(map.shape)
map_down[:, :-1] = map[:, 1:]
peaks_binary = np.logical_and.reduce(
(map >= map_left, map >= map_right, map >= map_up, map >= map_down,
map > param_['thre1']))
# peaks_binary = T.eq(
# peaks = zip(T.nonzero(peaks_binary)[0],T.nonzero(peaks_binary)[0])
peaks = zip(np.nonzero(peaks_binary)[1],
np.nonzero(peaks_binary)[0]) # note reverse
peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [
peaks_with_score[i] + (id[i], ) for i in range(len(id))
]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0] - 1]
candB = all_peaks[limbSeq[k][1] - 1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if (nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
vec = np.divide(vec, norm)
startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num))
vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(
vec_y, vec[1])
if norm == 0:
score_with_dist_prior = sum(score_midpts) / len(
score_midpts)
else:
score_with_dist_prior = sum(score_midpts) / len(
score_midpts) + min(
0.5 * oriImg.shape[0] / norm - 1, 0)
criterion1 = len(
np.nonzero(score_midpts > param_['thre2'])
[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([
i, j, score_with_dist_prior,
score_with_dist_prior + candA[i][2] + candB[j][2]
])
connection_candidate = sorted(connection_candidate,
key=lambda x: x[2],
reverse=True)
connection = np.zeros((0, 5))
for c in range(len(connection_candidate)):
i, j, s = connection_candidate[c][0:3]
if (i not in connection[:, 3] and j not in connection[:, 4]):
connection = np.vstack(
[connection, [candA[i][3], candB[j][3], s, i, j]])
if (len(connection) >= min(nA, nB)):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
#print 'candidate is ', candidate
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:, 0]
partBs = connection_all[k][:, 1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])): #= 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): #1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][
indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if (subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int),
2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
#print "found = 2"
membership = ((subset[j1] >= 0).astype(int) +
(subset[j2] >= 0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: #merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[
partBs[i].astype(int), 2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(
candidate[connection_all[k][i, :2].astype(int),
2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = []
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
deleteIdx.append(i)
#temp = np.delete(subset, deleteIdx, axis=0)
# canvas = cv2.imread(test_image) # B,G,R order
for i in range(18):
for j in range(len(all_peaks[i])):
cv2.circle(canvas,
all_peaks[i][j][0:2],
4,
colors[i],
thickness=-1)
stickwidth = 4
output_coordinates = np.zeros((len(subset), 18, 2))
for i in range(17):
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[i]) - 1]
if -1 in index:
continue
cur_canvas = canvas.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly(
(int(mY), int(mX)), (int(length / 2), stickwidth), int(angle),
0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
output_coordinates[n, i, 0] = mX
output_coordinates[n, i, 1] = mY
return canvas, output_coordinates, out1, out2, out3, out4, out5, out6
def generate_jointsmap(uv_coord, depth, width, height, channel=3):
canvas = np.zeros((height, width))
bones = [((17, 18), [130] * channel), ((18, 19), [140] * channel),
((19, 20), [150] * channel), ((1, 2), [10] * channel),
((2, 3), [20] * channel), ((3, 4), [30] * channel),
((5, 6), [40] * channel), ((6, 7), [50] * channel),
((7, 8), [60] * channel), ((9, 10), [70] * channel),
((10, 11), [80] * channel), ((11, 12), [90] * channel),
((13, 14), [100] * channel), ((14, 15), [110] * channel),
((15, 16), [120] * channel), ((0, 17), []), ((0, 1), []),
((0, 5), []), ((0, 9), []), ((0, 13), [])]
palm = []
for connection, _ in [
((0, 17), []),
((17, 1), []),
((1, 5), []),
((5, 9), []),
((9, 13), []),
((13, 0), []),
]:
coord1 = uv_coord[connection[0]]
palm.append([int(coord1[0]), int(coord1[1])])
# palm.append([int((coord1[0]-.5)* W_scale+ W_offset ), int(-(coord1[1]- .5)* H_scale+ H_offset)])
# print(palm)
palm_colors = [
depth[0], depth[17], depth[1], depth[5], depth[9], depth[13]
]
palm_colors = list(filter(lambda x: x >= 0, palm_colors))
# if len(palm_colors) == 6:
# binary_mask = np.zeros((height, width))
# cv2.fillConvexPoly(binary_mask, np.array([palm], dtype=np.int32), 1)
#
# avg_color_upper = np.average(palm_colors[2::])
# avg_color_lower = np.average(palm_colors[::2])
#
# Xs, Ys = np.array(palm).T
# Xmax, Xmin = np.max(Xs), np.min(Xs)
# Ymax, Ymin = np.max(Ys), np.min(Ys)
#
# orientation = None
# s_c = None # start_color
# d_c = None # end_color
# wrist = np.array([palm[0], palm[1]]).T
# if Xmax in wrist[0]:
# orientation = "leftRight"
# s_c = avg_color_upper
# d_c = avg_color_lower
# elif Xmin in wrist[0]:
# orientation = "RightLeft"
# s_c = avg_color_lower
# d_c = avg_color_upper
# elif Ymax in wrist[1]:
# orientation = "TopDown"
# s_c = avg_color_upper
# d_c = avg_color_lower
# else:
# orientation = "BottomUp"
# s_c = avg_color_lower
# d_c = avg_color_upper
#
# n_step = Xmax - Xmin if 'left' in orientation.lower() else Ymax - Ymin
#
# gradient_offset = np.abs(avg_color_lower - avg_color_upper) / n_step
#
# def add(x, y):
# return x+y
# def minus(x,y):
# return x-y
#
# color_operation = minus if s_c > d_c else add
#
# for i in range(int(n_step)):
# s_c = color_operation(s_c, i * gradient_offset)
# if 'left' in orientation.lower():
# canvas[:, Xmin + i] = s_c
# else:
# canvas[Ymin +i, :] = s_c
#
# canvas = np.multiply(canvas, binary_mask)
# else:
if len(palm_colors):
pass
# cv2.fillPoly(canvas, np.array([palm], dtype=np.int32), np.average(palm_colors))
for connection, color in bones:
temp_canvas = np.zeros(canvas.shape)
coord1 = uv_coord[connection[0]]
coord2 = uv_coord[connection[1]]
coords = np.stack([coord1, coord2])
colors = [depth[connection[0]], depth[connection[1]]]
if -1 in colors or len(colors) == 0:
continue
else:
color = np.average(colors)
# 0.5, 0.5 is the center
x = coords[:, 0]
y = coords[:, 1]
mX = x.mean()
mY = y.mean()
length = ((x[0] - x[1])**2 + (y[0] - y[1])**2)**0.5
angle = np.math.degrees(np.math.atan2(y[0] - y[1], x[0] - x[1]))
radius = 2
polygon = cv2.ellipse2Poly(
(int(mX), int(mY)), (int(length / 3), radius), int(angle), 0, 360,
1)
cv2.fillConvexPoly(temp_canvas, polygon, color)
canvas = np.maximum(canvas, temp_canvas)
return canvas
def segment_region(img, out_path, cnt, dist_per_pixel):
"""
Find regions containing cells in the image
:param img: Image object
:param dist_per_pixel: How big wide each pixel is
:return: List of regions
"""
area_per_pixel = dist_per_pixel * dist_per_pixel
img_h, img_w = img.shape[:2]
green = img[:, :, 1]
blurred = cv2.GaussianBlur(green, (BR_DEF, BR_DEF), 0)
# Apply a threshold to remove background noise
_, remaining = cv2.threshold(blurred, NT_MIN, 0, cv2.THRESH_TOZERO)
contours = []
noise_thresh = NT_MIN
while np.any(remaining):
# DEBUGGING
# cv2.imwrite('img/remaining.png', remaining)
# END
_, remaining = cv2.threshold(remaining, noise_thresh, 0,
cv2.THRESH_TOZERO)
_, prepared = cv2.threshold(remaining, noise_thresh, 255,
cv2.THRESH_BINARY)
_, remaining_contours, _ = cv2.findContours(prepared,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
too_small = []
for contour in remaining_contours:
# # DEBUGGING CODE ONLY - VERY SLOW
# cpy = img.copy()
# cv2.drawContours(cpy, [contour], -1, WHITE)
# cv2.imwrite('img/cntr.png', cpy)
# Determine smallest rectangle that surrounds contour
x, y, w, h = cv2.boundingRect(contour)
# Ignore cells touching edges
if x == 0 or y == 0 or x + w == img_w or y + h == img_h:
cv2.drawContours(remaining, [contour], 0, 0)
cv2.fillPoly(remaining, [contour], 0)
# when dealing with images containing only one cell,
# the cell might touch the edge
continue
area = w * h * area_per_pixel
if int(1.4 * MAX_AREA) <= area:
continue
elif int(0.1 * MIN_AREA) <= area:
cv2.drawContours(remaining, [contour], 0, 0)
cv2.fillPoly(remaining, [contour], 0)
if area <= int(0.6 * MIN_AREA):
too_small.append(contour)
else:
contours.append(contour)
else:
cv2.drawContours(remaining, [contour], 0, 0)
cv2.fillPoly(remaining, [contour], 0)
contours += merge_contours(too_small, area_per_pixel)
noise_thresh += 6
# Output the contours found
for contour in contours:
x, y, w, h = cv2.boundingRect(contour)
crypt = img[y:y + h, x:x + w].copy()
red_part = crypt[:, :, 2]
grn_chnl = crypt[:, :, 1]
centre_y = h // 2
centre_x = w // 2
grn_cntr = grn_chnl[centre_y - centre_y // 2:centre_y + centre_y // 2,
centre_x - centre_x // 2:centre_x + centre_x // 2]
_, red_nse_rmvd = cv2.threshold(red_part, 40, 255, cv2.THRESH_TOZERO)
_, grn_nse_rmvd = cv2.threshold(grn_cntr, 20, 255, cv2.THRESH_TOZERO)
# Red must account for > 2% of area to be considered significant
if cv2.countNonZero(red_nse_rmvd) > (w * h) / 20 and \
cv2.countNonZero(grn_nse_rmvd) > (w * h) / 16:
SCORING_AREA = 0.2
x_to_score = max(0, int(x - SCORING_AREA * w))
y_to_score = max(0, int(y - SCORING_AREA * h))
w_to_score = min(img_w - x_to_score,
int(w * (1 + 2 * SCORING_AREA)))
h_to_score = min(img_h - y_to_score,
int(h * (1 + 2 * SCORING_AREA)))
# Extract area we want our scorer to see
cell_to_score = img[y_to_score:y_to_score + h_to_score,
x_to_score:x_to_score + w_to_score].copy()
scoring_mask = np.zeros_like(cell_to_score)
half_w = w_to_score // 2
half_h = h_to_score // 2
ellipse = cv2.ellipse2Poly((half_w, half_h), (half_w, half_h), 0,
0, 360, 10)
cv2.fillPoly(scoring_mask, [ellipse], WHITE)
cell_to_score = cv2.bitwise_and(cell_to_score, scoring_mask)
cv2.imwrite(out_path + str(cnt) + '.png', cell_to_score)
cnt += 1
return cnt
def recent_events(self, events):
frame = events.get('frame')
if not frame:
return
for eye_index in self.showeyes:
requested_eye_frame_idx = self.eye_world_frame_map[eye_index][frame.index]
#1. do we need a new frame?
if requested_eye_frame_idx != self.eye_frames[eye_index].index:
# do we need to seek?
if requested_eye_frame_idx == self.eye_cap[eye_index].get_frame_index()+1:
# if we just need to seek by one frame, its faster to just read one and and throw it away.
_ = self.eye_cap[eye_index].get_frame()
if requested_eye_frame_idx != self.eye_cap[eye_index].get_frame_index():
# only now do I need to seek
self.eye_cap[eye_index].seek_to_frame(requested_eye_frame_idx)
# reading the new eye frame frame
try:
self.eye_frames[eye_index] = self.eye_cap[eye_index].get_frame()
except EndofVideoFileError:
logger.warning("Reached the end of the eye video for eye video {}.".format(eye_index))
else:
#our old frame is still valid because we are doing upsampling
pass
#2. dragging image
if self.drag_offset[eye_index] is not None:
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext()))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
self.pos[eye_index][0] = pos[0]+self.drag_offset[eye_index][0]
self.pos[eye_index][1] = pos[1]+self.drag_offset[eye_index][1]
else:
self.video_size = [round(self.eye_frames[eye_index].width*self.eye_scale_factor), round(self.eye_frames[eye_index].height*self.eye_scale_factor)]
#3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen
self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0))
#4. flipping images, converting to greyscale
eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling
eyeimage = cv2.resize(eye_gray,(0,0),fx=self.eye_scale_factor, fy=self.eye_scale_factor)
if self.mirror[str(eye_index)]:
eyeimage = np.fliplr(eyeimage)
if self.flip[str(eye_index)]:
eyeimage = np.flipud(eyeimage)
eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR)
if self.show_ellipses and events['pupil_positions']:
for pd in events['pupil_positions']:
if pd['id'] == eye_index and pd['timestamp'] == self.eye_frames[eye_index].timestamp:
el = pd['ellipse']
conf = int(pd.get('model_confidence', pd.get('confidence', 0.1)) * 255)
center = list(map(lambda val: int(self.eye_scale_factor*val), el['center']))
el['axes'] = tuple(map(lambda val: int(self.eye_scale_factor*val/2), el['axes']))
el['angle'] = int(el['angle'])
el_points = cv2.ellipse2Poly(tuple(center), el['axes'], el['angle'], 0, 360, 1)
if self.mirror[str(eye_index)]:
el_points = [(self.video_size[0] - x, y) for x, y in el_points]
center[0] = self.video_size[0] - center[0]
if self.flip[str(eye_index)]:
el_points = [(x, self.video_size[1] - y) for x, y in el_points]
center[1] = self.video_size[1] - center[1]
cv2.polylines(eyeimage, [np.asarray(el_points)], True, (0, 0, 255, conf), thickness=math.ceil(2*self.eye_scale_factor))
cv2.circle(eyeimage, tuple(center), int(5*self.eye_scale_factor), (0, 0, 255, conf), thickness=-1)
# 5. finally overlay the image
x, y = int(self.pos[eye_index][0]), int(self.pos[eye_index][1])
transparent_image_overlay((x, y), eyeimage, frame.img, self.alpha)
def plot_poses(img,
skeletons,
config=plt_config('coco'),
save_path=None,
dataset_name='coco'):
# std = torch.tensor([0.229, 0.224, 0.225]).unsqueeze(-1).unsqueeze(-1)
# mean = torch.tensor([0.485, 0.456, 0.406]).unsqueeze(-1).unsqueeze(-1)
# # input = input.detach().cpu()
# #if dataset_name == 'coco':
# img = img * std + mean
# img = img.permute(1, 2, 0).detach().numpy() # [H,W,3]
# colors = [[255, 0, 0],
# [155, 85, 0],
# [255, 170, 0],
# [255, 55, 0],
# [17, 25, 10],
# [85, 55, 0],
# [0, 255, 0],
# [0, 255, 5],
# [0, 255, 170],
# [0, 255, 25],
# [0, 17, 255],
# [0, 185, 255],
# [0, 0, 255],
# [125, 0, 255],
# [170, 0, 25],
# [25, 0, 255],
# [255, 0, 170],
# [255, 0, 85],
# [0, 170, 255],
# [0, 85, 255],
# [0, 0, 255],
# [85, 0, 255],
# [70, 0, 255],
# [25, 0, 255],
# [255, 0, 70],
# [55, 10, 5]]
cmap = matplotlib.cm.get_cmap('hsv')
canvas = img.copy()
n_kpt = config.n_kpt
for i in range(n_kpt):
rgba = np.array(cmap(1 - i / n_kpt - 1. / n_kpt * 2))
rgba[0:3] *= 255
for j in range(len(skeletons)):
if len(skeletons[j][i]) > 2 and skeletons[j][i, 2] > 0:
cv2.circle(canvas,
tuple(skeletons[j][i, 0:2].astype('int32')),
3, (255, 255, 255),
thickness=-1)
# to_plot = cv2.addWeighted(img, 0.3, canvas, 0.7, 0)
stickwidth = 2
for i in range(len(config.EDGES)):
for j in range(len(skeletons)):
edge = config.EDGES[i][0]
color = config.EDGES[i][1]
if len(skeletons[j][edge[0]]) > 2:
if skeletons[j][edge[0], 2] == 0 or skeletons[j][edge[1],
2] == 0:
continue
cur_canvas = canvas.copy()
X = [skeletons[j][edge[0], 1], skeletons[j][edge[1], 1]]
Y = [skeletons[j][edge[0], 0], skeletons[j][edge[1], 0]]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly(
(int(mY), int(mX)), (int(length / 2), stickwidth), int(angle),
0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, color)
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
return canvas
def eye(pupil_queue, timebase, pipe_to_world, is_alive_flag, user_dir, version, eye_id, cap_src):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
is_alive = Is_Alive_Manager(is_alive_flag)
with is_alive:
import logging
# Set up root logger for this process before doing imports of logged modules.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(user_dir,'eye%s.log'%eye_id),mode='w')
# fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logger.level+10)
# create formatter and add it to the handlers
formatter = logging.Formatter('Eye'+str(eye_id)+' Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter('EYE'+str(eye_id)+' Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
#silence noisy modules
logging.getLogger("OpenGL").setLevel(logging.ERROR)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# We deferr the imports becasue of multiprocessing.
# Otherwise the world process each process also loads the other imports.
#general imports
import numpy as np
import cv2
#display
import glfw
from pyglui import ui,graph,cygl
from pyglui.cygl.utils import draw_points, RGBA, draw_polyline, Named_Texture, Sphere
import OpenGL.GL as gl
from gl_utils import basic_gl_setup,adjust_gl_view, clear_gl_screen ,make_coord_system_pixel_based,make_coord_system_norm_based, make_coord_system_eye_camera_based
from ui_roi import UIRoi
#monitoring
import psutil
import math
# helpers/utils
from file_methods import Persistent_Dict
from version_utils import VersionFormat
from methods import normalize, denormalize, Roi, timer
from video_capture import autoCreateCapture, FileCaptureError, EndofVideoFileError, CameraCaptureError
from av_writer import JPEG_Writer,AV_Writer
# Pupil detectors
from pupil_detectors import Detector_2D, Detector_3D
pupil_detectors = {Detector_2D.__name__:Detector_2D,Detector_3D.__name__:Detector_3D}
#UI Platform tweaks
if platform.system() == 'Linux':
scroll_factor = 10.0
window_position_default = (600,300*eye_id)
elif platform.system() == 'Windows':
scroll_factor = 1.0
window_position_default = (600,31+300*eye_id)
else:
scroll_factor = 1.0
window_position_default = (600,300*eye_id)
#g_pool holds variables for this process
g_pool = Global_Container()
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = version
g_pool.app = 'capture'
g_pool.pupil_queue = pupil_queue
g_pool.timebase = timebase
# Callback functions
def on_resize(window,w, h):
if not g_pool.iconified:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
glfw.glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
g_pool.gui.update_char(char)
def on_iconify(window,iconified):
g_pool.iconified = iconified
def on_button(window,button, action, mods):
if g_pool.display_mode == 'roi':
if action == glfw.GLFW_RELEASE and g_pool.u_r.active_edit_pt:
g_pool.u_r.active_edit_pt = False
return # if the roi interacts we dont what the gui to interact as well
elif action == glfw.GLFW_PRESS:
pos = glfw.glfwGetCursorPos(window)
pos = normalize(pos,glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height)) # Position in img pixels
if g_pool.u_r.mouse_over_edit_pt(pos,g_pool.u_r.handle_size+40,g_pool.u_r.handle_size+40):
return # if the roi interacts we dont what the gui to interact as well
g_pool.gui.update_button(button,action,mods)
def on_pos(window,x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor,y*hdpi_factor)
if g_pool.u_r.active_edit_pt:
pos = normalize((x,y),glfw.glfwGetWindowSize(main_window))
if g_pool.flip:
pos = 1-pos[0],1-pos[1]
pos = denormalize(pos,(frame.width,frame.height) )
g_pool.u_r.move_vertex(g_pool.u_r.active_pt_idx,pos)
def on_scroll(window,x,y):
g_pool.gui.update_scroll(x,y*scroll_factor)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye%s'%eye_id))
if session_settings.get("version",VersionFormat('0.0')) < g_pool.version:
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
# Initialize capture
cap = autoCreateCapture(cap_src, timebase=g_pool.timebase)
default_settings = {'frame_size':(640,480),'frame_rate':60}
previous_settings = session_settings.get('capture_settings',None)
if previous_settings and previous_settings['name'] == cap.name:
cap.settings = previous_settings
else:
cap.settings = default_settings
# Test capture
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
#signal world that we are ready to go
# pipe_to_world.send('eye%s process ready'%eye_id)
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.iconified = False
g_pool.capture = cap
g_pool.flip = session_settings.get('flip',False)
g_pool.display_mode = session_settings.get('display_mode','camera_image')
g_pool.display_mode_info_text = {'camera_image': "Raw eye camera image. This uses the least amount of CPU power",
'roi': "Click and drag on the blue circles to adjust the region of interest. The region should be as small as possible, but large enough to capture all pupil movements.",
'algorithm': "Algorithm display mode overlays a visualization of the pupil detection parameters on top of the eye video. Adjust parameters within the Pupil Detection menu below."}
g_pool.u_r = UIRoi(frame.img.shape)
g_pool.u_r.set(session_settings.get('roi',g_pool.u_r.get()))
def on_frame_size_change(new_size):
g_pool.u_r = UIRoi((new_size[1],new_size[0]))
cap.on_frame_size_change = on_frame_size_change
writer = None
pupil_detector_settings = session_settings.get('pupil_detector_settings',None)
last_pupil_detector = pupil_detectors[session_settings.get('last_pupil_detector',Detector_2D.__name__)]
g_pool.pupil_detector = last_pupil_detector(g_pool,pupil_detector_settings)
# UI callback functions
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_display_mode_info(val):
g_pool.display_mode = val
g_pool.display_mode_info.text = g_pool.display_mode_info_text[val]
def set_detector(new_detector):
g_pool.pupil_detector.cleanup()
g_pool.pupil_detector = new_detector(g_pool)
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# Initialize glfw
glfw.glfwInit()
title = "eye %s"%eye_id
width,height = session_settings.get('window_size',(frame.width, frame.height))
main_window = glfw.glfwCreateWindow(width,height, title, None, None)
window_pos = session_settings.get('window_position',window_position_default)
glfw.glfwSetWindowPos(main_window,window_pos[0],window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
# gl_state settings
basic_gl_setup()
g_pool.image_tex = Named_Texture()
g_pool.image_tex.update_from_frame(frame)
glfw.glfwSwapInterval(0)
sphere = Sphere(20)
#setup GUI
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale',1)
g_pool.sidebar = ui.Scrolling_Menu("Settings",pos=(-300,0),size=(0,0),header_pos='left')
general_settings = ui.Growing_Menu('General')
general_settings.append(ui.Slider('scale',g_pool.gui, setter=set_scale,step = .05,min=1.,max=2.5,label='Interface Size'))
general_settings.append(ui.Button('Reset window size',lambda: glfw.glfwSetWindowSize(main_window,frame.width,frame.height)) )
general_settings.append(ui.Switch('flip',g_pool,label='Flip image display'))
general_settings.append(ui.Selector('display_mode',g_pool,setter=set_display_mode_info,selection=['camera_image','roi','algorithm'], labels=['Camera Image', 'ROI', 'Algorithm'], label="Mode") )
g_pool.display_mode_info = ui.Info_Text(g_pool.display_mode_info_text[g_pool.display_mode])
general_settings.append(g_pool.display_mode_info)
g_pool.sidebar.append(general_settings)
g_pool.gui.append(g_pool.sidebar)
detector_selector = ui.Selector('pupil_detector',getter = lambda: g_pool.pupil_detector.__class__ ,setter=set_detector,selection=[Detector_2D, Detector_3D],labels=['C++ 2d detector', 'C++ 3d detector'], label="Detection method")
general_settings.append(detector_selector)
# let detector add its GUI
g_pool.pupil_detector.init_gui(g_pool.sidebar)
# let the camera add its GUI
g_pool.capture.init_gui(g_pool.sidebar)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window,on_resize)
glfw.glfwSetWindowIconifyCallback(main_window,on_iconify)
glfw.glfwSetKeyCallback(main_window,on_key)
glfw.glfwSetCharCallback(main_window,on_char)
glfw.glfwSetMouseButtonCallback(main_window,on_button)
glfw.glfwSetCursorPosCallback(main_window,on_pos)
glfw.glfwSetScrollCallback(main_window,on_scroll)
#set the last saved window size
on_resize(main_window, *glfw.glfwGetWindowSize(main_window))
# load last gui configuration
g_pool.gui.configuration = session_settings.get('ui_config',{})
#set up performance graphs
pid = os.getpid()
ps = psutil.Process(pid)
ts = frame.timestamp
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20,130)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140,130)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f FPS"
#create a timer to control window update frequency
window_update_timer = timer(1/60.)
def window_should_update():
return next(window_update_timer)
# Event loop
while not glfw.glfwWindowShouldClose(main_window):
if pipe_to_world.poll():
cmd = pipe_to_world.recv()
if cmd == 'Exit':
break
elif cmd == "Ping":
pipe_to_world.send("Pong")
command = None
else:
command,payload = cmd
if command == 'Set_Detection_Mapping_Mode':
if payload == '3d':
if not isinstance(g_pool.pupil_detector,Detector_3D):
set_detector(Detector_3D)
detector_selector.read_only = True
else:
set_detector(Detector_2D)
detector_selector.read_only = False
else:
command = None
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
cap.seek_to_frame(0)
frame = cap.get_frame()
#update performace graphs
t = frame.timestamp
dt,ts = t-ts,t
try:
fps_graph.add(1./dt)
except ZeroDivisionError:
pass
cpu_graph.update()
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if 'Rec_Start' == command:
record_path,raw_mode = payload
logger.info("Will save eye video to: %s"%record_path)
timestamps_path = os.path.join(record_path, "eye%s_timestamps.npy"%eye_id)
if raw_mode and frame.jpeg_buffer:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = JPEG_Writer(video_path,cap.frame_rate)
else:
video_path = os.path.join(record_path, "eye%s.mp4"%eye_id)
writer = AV_Writer(video_path,cap.frame_rate)
timestamps = []
elif 'Rec_Stop' == command:
logger.info("Done recording.")
writer.release()
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
if writer:
writer.write_video_frame(frame)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = g_pool.pupil_detector.detect(frame, g_pool.u_r, g_pool.display_mode == 'algorithm')
result['id'] = eye_id
# stream the result
g_pool.pupil_queue.put(result)
# GL drawing
if window_should_update():
if not g_pool.iconified:
glfw.glfwMakeContextCurrent(main_window)
clear_gl_screen()
# switch to work in normalized coordinate space
if g_pool.display_mode == 'algorithm':
g_pool.image_tex.update_from_ndarray(frame.img)
elif g_pool.display_mode in ('camera_image','roi'):
g_pool.image_tex.update_from_ndarray(frame.gray)
else:
pass
make_coord_system_norm_based(g_pool.flip)
g_pool.image_tex.draw()
window_size = glfw.glfwGetWindowSize(main_window)
make_coord_system_pixel_based((frame.height,frame.width,3),g_pool.flip)
if result['method'] == '3d c++':
eye_ball = result['projected_sphere']
try:
pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])),
(int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)),
int(eye_ball['angle']),0,360,8)
except ValueError as e:
pass
else:
draw_polyline(pts,2,RGBA(0.,.9,.1,result['model_confidence']) )
if result['confidence'] >0:
if result.has_key('ellipse'):
pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])),
(int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)),
int(result['ellipse']['angle']),0,360,15)
confidence = result['confidence'] * 0.7 #scale it a little
draw_polyline(pts,1,RGBA(1.,0,0,confidence))
draw_points([result['ellipse']['center']],size=20,color=RGBA(1.,0.,0.,confidence),sharpness=1.)
# render graphs
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
graph.pop_view()
# render GUI
g_pool.gui.update()
#render the ROI
if g_pool.display_mode == 'roi':
g_pool.u_r.draw(g_pool.gui.scale)
#update screen
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
g_pool.pupil_detector.visualize() #detector decides if we visualize or not
# END while running
# in case eye recording was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
glfw.glfwRestoreWindow(main_window) #need to do this for windows os
# save session persistent settings
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['roi'] = g_pool.u_r.get()
session_settings['flip'] = g_pool.flip
session_settings['display_mode'] = g_pool.display_mode
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['capture_settings'] = g_pool.capture.settings
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings['last_pupil_detector'] = g_pool.pupil_detector.__class__.__name__
session_settings['pupil_detector_settings'] = g_pool.pupil_detector.get_settings()
session_settings.close()
g_pool.pupil_detector.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
glfw.glfwTerminate()
cap.close()
logger.debug("Process done")
def predict(self, test_image, idx):
#test_image = '../sample_image/ski.jpg'
#test_image = '../sample_image/upper.jpg'
#test_image = '../sample_image/upper2.jpg'
oriImg = cv.imread(test_image) # B,G,R order
param, model = config_reader()
multiplier = [
x * model['boxsize'] / oriImg.shape[0]
for x in param['scale_search']
]
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 15))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 26))
# first figure shows padded images
for m in range(len(multiplier)):
scale = multiplier[m]
imageToTest = cv.resize(oriImg, (0, 0),
fx=scale,
fy=scale,
interpolation=cv.INTER_CUBIC)
imageToTest_padded, pad = util.padRightDownCorner(
imageToTest, model['stride'], model['padValue'])
print imageToTest_padded.shape
self.net.blobs['data'].reshape(*(1, 3, imageToTest_padded.shape[0],
imageToTest_padded.shape[1]))
#net.forward() # dry run
self.net.blobs['data'].data[...] = np.transpose(
np.float32(imageToTest_padded[:, :, :, np.newaxis]),
(3, 2, 0, 1)) / 256 - 0.5
start_time = time.time()
output_blobs = self.net.forward()
print('At scale %d, The CNN took %.2f ms.' %
(m, 1000 * (time.time() - start_time)))
# extract outputs, resize, and remove padding
heatmap = np.transpose(
np.squeeze(self.net.blobs[output_blobs.keys()[1]].data),
(1, 2, 0)) # output 1 is heatmaps
heatmap = cv.resize(heatmap, (0, 0),
fx=model['stride'],
fy=model['stride'],
interpolation=cv.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
heatmap = cv.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv.INTER_CUBIC)
paf = np.transpose(
np.squeeze(self.net.blobs[output_blobs.keys()[0]].data),
(1, 2, 0)) # output 0 is PAFs
paf = cv.resize(paf, (0, 0),
fx=model['stride'],
fy=model['stride'],
interpolation=cv.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
paf = cv.resize(paf, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv.INTER_CUBIC)
# visualization
for k in range(14):
'''
axarr2[k,m].imshow(oriImg[:,:,[2,1,0]])
ax2 = axarr2[k,m].imshow(heatmap[:,:,k], alpha=.5) # right wrist
axarr2[k,m].set_title('Heatmaps (Rwri): scale %d' % m)
'''
'''
plt.imshow(oriImg[:,:,[2,1,0]])
plt.imshow(heatmap[:,:,k], alpha=.5) # right wrist
plt.savefig('{}_{}.png'.format(k, m))
'''
'''
axarr3.flat[m].imshow(oriImg[:,:,[2,1,0]])
ax3x = axarr3.flat[m].imshow(paf[:,:,16], alpha=.5) # right elbow
axarr3.flat[m].set_title('PAFs (x comp. of Rwri to Relb): scale %d' % m)
axarr3.flat[len(multiplier) + m].imshow(oriImg[:,:,[2,1,0]])
ax3y = axarr3.flat[len(multiplier) + m].imshow(paf[:,:,17], alpha=.5) # right wrist
axarr3.flat[len(multiplier) + m].set_title('PAFs (y comp. of Relb to Rwri): scale %d' % m)
'''
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
'''
for i in range(14):
plt.imshow(oriImg[:,:,[2,1,0]])
plt.imshow(heatmap_avg[:,:,i], alpha=.5)
plt.savefig(str(i))
'''
'''
for i in range(13):
U = paf_avg[:,:,i*2] * -1
V = paf_avg[:,:,i*2+1]
X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0]))
M = np.zeros(U.shape, dtype='bool')
M[U**2 + V**2 < 0.5 * 0.5] = True
U = ma.masked_array(U, mask=M)
V = ma.masked_array(V, mask=M)
# 1
plt.figure()
plt.imshow(oriImg[:,:,[2,1,0]], alpha = .5)
s = 5
Q = plt.quiver(X[::s,::s], Y[::s,::s], U[::s,::s], V[::s,::s],
scale=50, headaxislength=4, alpha=.5, width=0.001, color='r')
plt.savefig('_{}.png'.format(i))
'''
'''
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(20, 20)
'''
print heatmap_avg.shape
#plt.imshow(heatmap_avg[:,:,2])
all_peaks = []
peak_counter = 0
for part in range(15 - 1):
x_list = []
y_list = []
map_ori = heatmap_avg[:, :, part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:, :] = map[:-1, :]
map_right = np.zeros(map.shape)
map_right[:-1, :] = map[1:, :]
map_up = np.zeros(map.shape)
map_up[:, 1:] = map[:, :-1]
map_down = np.zeros(map.shape)
map_down[:, :-1] = map[:, 1:]
peaks_binary = np.logical_and.reduce(
(map >= map_left, map >= map_right, map >= map_up,
map >= map_down, map > param['thre1']))
peaks = zip(
np.nonzero(peaks_binary)[1],
np.nonzero(peaks_binary)[0]) # note reverse
peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [
peaks_with_score[i] + (id[i], ) for i in range(len(id))
]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
# find connection in the specified sequence, center 29 is in the position 15
limbSeq = [[13, 14], [14, 4], [4, 5], [5, 6], [14, 1], [1, 2], [2, 3],
[14, 10], [10, 11], [11, 12], [14, 7], [7, 8], [8, 9]]
# the middle joints heatmap correpondence
mapIdx = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13],
[14, 15], [16, 17], [18, 19], [20, 21], [22, 23], [24, 25]]
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf_avg[:, :, [x for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0] - 1]
candB = all_peaks[limbSeq[k][1] - 1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if (nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
vec = np.divide(vec, norm)
startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num))
vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(
vec_x, vec[0]) + np.multiply(vec_y, vec[1])
score_with_dist_prior = sum(score_midpts) / len(
score_midpts) + min(
0.5 * oriImg.shape[0] / norm - 1, 0)
criterion1 = len(
np.nonzero(score_midpts > param['thre2'])
[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([
i, j, score_with_dist_prior,
score_with_dist_prior + candA[i][2] +
candB[j][2]
])
connection_candidate = sorted(connection_candidate,
key=lambda x: x[2],
reverse=True)
connection = np.zeros((0, 5))
for c in range(len(connection_candidate)):
i, j, s = connection_candidate[c][0:3]
if (i not in connection[:, 3]
and j not in connection[:, 4]):
connection = np.vstack(
[connection, [candA[i][3], candB[j][3], s, i, j]])
if (len(connection) >= min(nA, nB)):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array(
[item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:, 0]
partBs = connection_all[k][:, 1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])): #= 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): #1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][
indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if (subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[
partBs[i].astype(int),
2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
print "found = 2"
membership = ((subset[j1] >= 0).astype(int) +
(subset[j2] >= 0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: #merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[
partBs[i].astype(int),
2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(
candidate[connection_all[k][i, :2].astype(int),
2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = []
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
# visualize
b = [255, 0, 0]
r = [0, 0, 255]
g = [0, 255, 0]
colors = [b, b, b, r, r, r, b, b, b, r, r, r, g, g]
colors2 = [g, r, r, r, b, b, b, r, r, r, b, b, b]
'''
[[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
[0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
[170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
'''
cmap = matplotlib.cm.get_cmap('hsv')
canvas = cv.imread(test_image) # B,G,R order
for i in range(14):
rgba = np.array(cmap(1 - i / 18. - 1. / 36))
rgba[0:3] *= 255
for j in range(len(all_peaks[i])):
cv.circle(canvas,
all_peaks[i][j][0:2],
4,
colors[i],
thickness=-1)
to_plot = cv.addWeighted(oriImg, 0.3, canvas, 0.7, 0)
plt.imshow(to_plot[:, :, [2, 1, 0]])
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(12, 12)
# visualize 2
stickwidth = 4
for i in range(13):
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[i]) - 1]
if -1 in index:
continue
cur_canvas = canvas.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv.ellipse2Poly((int(mY), int(mX)),
(int(length / 2), stickwidth),
int(angle), 0, 360, 1)
cv.fillConvexPoly(cur_canvas, polygon, colors2[i % 13])
canvas = cv.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
plt.imshow(canvas[:, :, [2, 1, 0]])
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(12, 12)
plt.savefig('predict_{}Crop'.format(idx))