def estimate(image_file, mode='nose_2eyes'):
im = cv2.imread(image_file)
landmarks = face_68_landmarks(im)
height, width = im.shape[:2]
print(height, width)
colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (0, 255, 255), (255, 255, 0), (255, 125, 125)]
pose_estimator = HeadPoseEstimator(image_size=(height, width), mode=mode)
for marks in landmarks:
# for pnt in marks:
# cv2.circle(im, (int(pnt[0]), int(pnt[1])), 1, (0, 255, 0), 2, cv2.LINE_AA)
image_points = get_points_from_landmarks(marks, mode)
print('========len======== : ', len(image_points))
print('======== ======== : ', image_points)
rotation_vector, translation_vector = pose_estimator.solve_pose(image_points)
print('-------------------------\n', rotation_vector, '||||\n', translation_vector,
'\n-------------------------\n')
end_points_2d = pose_estimator.projection(rotation_vector, translation_vector)
for i, pnt in enumerate(image_points.tolist()):
cv2.circle(im, (int(pnt[0]), int(pnt[1])), 1, colors[i % 6], 3, cv2.LINE_AA)
end_points_2d = np.array(end_points_2d).astype(np.int).tolist()
cv2.line(im, tuple(end_points_2d[5]), tuple(end_points_2d[6]), (0, 255, 0))
cv2.line(im, tuple(end_points_2d[6]), tuple(end_points_2d[7]), (255, 0, 0))
cv2.line(im, tuple(end_points_2d[2]), tuple(end_points_2d[6]), (0, 0, 255))
return im
def pose_estimate():
landmarks = face_68_landmarks(frame, cnn=False)
height, width = frame.shape[:2]
print(height, width)
pose_estimator = HeadPoseEstimator(image_size=(height, width),
mode='nose_eyes_mouth')
for marks in landmarks:
image_points = get_points_from_landmarks(marks, mode='nose_eyes_mouth')
rotation_vector, translation_vector = pose_estimator.solve_pose(
image_points)
print('-----------------------------------')
print(rotation_vector, '===\n', translation_vector)
nose_end_points_2d = pose_estimator.projection(rotation_vector,
translation_vector)
for pnt in image_points.tolist():
cv2.circle(frame, (int(pnt[0]), int(pnt[1])), 1, (0, 255, 0), 1,
cv2.LINE_AA)
for pnt in nose_end_points_2d.tolist():
cv2.circle(frame, (int(pnt[0]), int(pnt[1])), 1, (0, 0, 255), 1,
cv2.LINE_AA)
nose_end_points_2d = np.array(nose_end_points_2d).astype(
np.int).tolist()
cv2.line(frame, tuple(nose_end_points_2d[5]),
tuple(nose_end_points_2d[6]), (0, 255, 0))
cv2.line(frame, tuple(nose_end_points_2d[6]),
tuple(nose_end_points_2d[7]), (255, 0, 0))
cv2.line(frame, tuple(nose_end_points_2d[2]),
tuple(nose_end_points_2d[6]), (0, 0, 255))
from face_detector import MxnetDetectionModel
import os
os.chdir(os.path.dirname(__file__))
gpu_ctx = -1
video = sys.argv[1]
YAW_THD = 45
cap = cv2.VideoCapture(video)
fd = MxnetDetectionModel("../weights/16and32", 0, .6, gpu=gpu_ctx)
fa = CoordinateAlignmentModel('../weights/2d106det', 0, gpu=gpu_ctx)
gs = IrisLocalizationModel("../weights/iris_landmark.tflite")
hp = HeadPoseEstimator("../weights/object_points.npy", cap.get(3), cap.get(4))
counter = 0
while True:
ret, frame = cap.read()
if not ret:
break
# frame = frame[:480, 380:920, :] # dress
# frame = cv2.resize(frame, (960, 1080))
bboxes = fd.detect(frame)
for landmarks in fa.get_landmarks(frame, bboxes, calibrate=True):
def process(input_image, params, model_params):
oriImg = cv2.imread(input_image) # B,G,R order
multiplier = [
x * model_params['boxsize'] / oriImg.shape[0]
for x in params['scale_search']
]
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
for m in range(len(multiplier)):
scale = multiplier[m]
imageToTest = cv2.resize(oriImg, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.pad_right_down_corner(
imageToTest, model_params['stride'], model_params['padValue'])
input_img = np.transpose(
np.float32(imageToTest_padded[:, :, :, np.newaxis]),
(3, 0, 1, 2)) # required shape (1, width, height, channels)
output_blobs = model.predict(input_img)
# extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv2.INTER_CUBIC)
paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv2.INTER_CUBIC)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
all_peaks = []
peak_counter = 0
for part in range(18):
map_ori = heatmap_avg[:, :, part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:, :] = map[:-1, :]
map_right = np.zeros(map.shape)
map_right[:-1, :] = map[1:, :]
map_up = np.zeros(map.shape)
map_up[:, 1:] = map[:, :-1]
map_down = np.zeros(map.shape)
map_down[:, :-1] = map[:, 1:]
peaks_binary = np.logical_and.reduce(
(map >= map_left, map >= map_right, map >= map_up, map >= map_down,
map > params['thre1']))
peaks = list(
zip(np.nonzero(peaks_binary)[1],
np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [
peaks_with_score[i] + (id[i], ) for i in range(len(id))
]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0] - 1]
candB = all_peaks[limbSeq[k][1] - 1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if nA != 0 and nB != 0:
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
# failure case when 2 body parts overlaps
if norm == 0:
continue
vec = np.divide(vec, norm)
startend = list(
zip(np.linspace(candA[i][0], candB[j][0], num=mid_num),
np.linspace(candA[i][1], candB[j][1],
num=mid_num)))
vec_x = np.array(
[score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array(
[score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(
vec_y, vec[1])
score_with_dist_prior = sum(
score_midpts) / len(score_midpts) + min(
0.5 * oriImg.shape[0] / norm - 1, 0)
criterion1 = len(
np.nonzero(score_midpts > params['thre2'])
[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([
i, j, score_with_dist_prior,
score_with_dist_prior + candA[i][2] + candB[j][2]
])
connection_candidate = sorted(connection_candidate,
key=lambda x: x[2],
reverse=True)
connection = np.zeros((0, 5))
for c in range(len(connection_candidate)):
i, j, s = connection_candidate[c][0:3]
if i not in connection[:, 3] and j not in connection[:, 4]:
connection = np.vstack(
[connection, [candA[i][3], candB[j][3], s, i, j]])
if len(connection) >= min(nA, nB):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:, 0]
partBs = connection_all[k][:, 1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])): # = 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): # 1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][
indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if subset[j][indexB] != partBs[i]:
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int),
2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
membership = ((subset[j1] >= 0).astype(int) +
(subset[j2] >= 0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: # merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[
partBs[i].astype(int), 2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + \
connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = []
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
canvas = cv2.imread(input_image) # B,G,R order
print('---------len subset: {}'.format(len(subset)))
persons_limbs = []
for n in range(len(subset)):
limbs = []
for i in range(17):
index = subset[n][np.array(limbSeq[i]) - 1]
if -1 in index:
limbs.append([])
continue
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
limbs.append([(X[0], Y[0]), (X[1], Y[1])])
# cv2.line(canvas, (int(Y[0]), int(X[0])), (int(Y[1]), int(X[1])), colors[n % 18], 2)
persons_limbs.append(limbs)
height, width = oriImg.shape[:2]
head_pose = HeadPoseEstimator((height, width), mode='nose_eyes_ears')
for limbs in persons_limbs:
body_pose = geo.BodyPose.from_connected_body_parts(limbs)
# geo.draw_body_pose_key_points(body_pose, canvas, colors)
# if body_pose.is_hands_up(threshold=30):
# print('----------hand up----------')
# x1, y1, x2, y2 = body_pose.mbr()
# cv2.rectangle(canvas, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2)
# cv2.putText(canvas, 'Hand up', (int(x1), int(y1) - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 0), 2)
# if body_pose.left_elbow is not None:
# print('===left_elbow===: {}'.format(body_pose.nose))
# cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.left_elbow), 4, (0, 255, 255), thickness=-1)
#
# if body_pose.nose is not None:
# print('===nose===: {}'.format(body_pose.nose))
# cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.nose), 4, (0, 255, 0), thickness=-1)
# if body_pose.left_wrist is not None:
# print('===left_wrist===: {}'.format(body_pose.left_wrist))
# cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.left_wrist), 4, (255, 0, 0), thickness=-1)
# if body_pose.right_wrist is not None:
# print('===right_wrist===: {}'.format(body_pose.right_wrist))
# cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.right_wrist), 4, (0, 0, 255), thickness=-1)
body_pose.head_direction(canvas, offset2nose=30)
# print(body_pose.nose_2eyes())
# if body_pose.head_key_points() is not None:
# rotation_vector, translation_vector = head_pose.solve_pose(body_pose.head_key_points())
# print('r: {}, t: {}'.format(rotation_vector, translation_vector))
# points_2d = head_pose.projection(rotation_vector, translation_vector)
# points_2d = np.array(points_2d).astype(np.int).tolist()
# print('points: {}'.format(points_2d))
# geo.draw_head_pose(canvas, points_2d)
return canvas