def predict(self, im, boxes, landmarks):
img_rotated, _, (angle, center) = ratate(boxes, landmarks, img=im)
image = img_rotated.copy()
# bbox = boxes
# cv2.putText(image,str(np.round(bbox[4],2)),(int(bbox[0]),int(bbox[1])),cv2.FONT_HERSHEY_TRIPLEX,1,color=(255,0,255))
# cv2.rectangle(image, (int(bbox[0]),int(bbox[1])),(int(bbox[2]),int(bbox[3])),(0,0,255))
# cv2.imwrite('rotated.jpg', image)
h, w, c = img_rotated.shape
W, H = boxes[2:4] - boxes[0:2] + 1
# print(angle)
rate = 1.0
# if angle > 10:
# rate = 1.2
# elif angle > 15:
# rate = 2
# face data
x1, y1, x2, y2, _ = boxes
dx1, dy1, dx2, dy2 = W * 0.2 * rate, (
0.1 * rate - 0.2) * H, W * 0.2 * rate, H * 0.2 * rate
x1 = max(0, int(x1 - dx1 + 0.5))
y1 = max(0, int(y1 - dy1 + 0.5))
x2 = min(w - 1, int(x2 + dx2 + 0.5))
y2 = min(h - 1, int(y2 + dy2 + 0.5))
# cv2.rectangle(image, (int(x1),int(y1)),(int(x2),int(y2)),(255,0,0))
# cv2.imwrite('rotated.jpg', image)
# face_box = (x1,y1,x2,y2)
cropped_im = img_rotated[y1:y2, x1:x2, :]
# cropped_im = cv2.resize(cropped_im, (38,38))
cropped_ims = np.zeros((1, self.net_size, self.net_size, 3),
dtype=np.float32)
cropped_ims[0, :, :, :] = (cv2.resize(cropped_im,
(self.net_size, self.net_size)) -
127.5) / 128
landmark = self.net.predict(cropped_ims)
wh = np.asarray([[x2 - x1 + 1, y2 - y1 + 1]], dtype=np.float32)
xy = np.asarray([[x1, y1]], dtype=np.float32)
landmark = landmark.reshape(-1, 2) * wh + xy
rot_mat = cv2.getRotationMatrix2D(center, -angle, 1)
landmark_ = np.asarray([
(rot_mat[0][0] * x + rot_mat[0][1] * y + rot_mat[0][2],
rot_mat[1][0] * x + rot_mat[1][1] * y + rot_mat[1][2])
for (x, y) in landmark
])
return landmark
def predict(self, im, boxes, landmarks):
img_rotated, (landmark_pred_rotated, ), (angle, center) = ratate(
boxes, landmarks, points=[landmarks], img=im)
h, w, c = img_rotated.shape
landmark_left_eye = landmark_pred_rotated[0]
landmark_left_mouse = landmark_pred_rotated[3]
landmark_right_mouse = landmark_pred_rotated[4]
W = landmark_right_mouse[0] - landmark_left_mouse[0] + 1
H = landmark_left_mouse[1] - landmark_left_eye[1] + 1
dx1, dy1, dx2, dy2 = W * 0.35, H * 0.45, W * 0.35, H * 0.6
# if x1 < 0.35 and y1 < 0.45 and x2 < 0.35 and y2 < 0.6:
x1_, y1_ = landmark_left_mouse
x2_, y2_ = landmark_right_mouse
x1 = min(x1_, x2_)
x2 = max(x1_, x2_)
y1 = min(y1_, y2_)
y2 = max(y1_, y2_)
x1 = max(0, int(x1 - dx1 + 0.5))
y1 = max(0, int(y1 - dy1 + 0.5))
x2 = min(w - 1, int(x2 + dx2 + 0.5))
y2 = min(h - 1, int(y2 + dy2 + 0.5))
cropped_im = img_rotated[y1:y2, x1:x2, :]
# cropped_im = cv2.resize(cropped_im, (38,38))
cropped_ims = np.zeros((1, self.net_size, self.net_size, 3),
dtype=np.float32)
cropped_ims[0, :, :, :] = (cv2.resize(cropped_im,
(self.net_size, self.net_size)) -
127.5) / 128
landmark = self.net.predict(cropped_ims)
wh = np.asarray([[x2 - x1 + 1, y2 - y1 + 1]], dtype=np.float32)
xy = np.asarray([[x1, y1]], dtype=np.float32)
landmark = landmark.reshape(-1, 2) * wh + xy
rot_mat = cv2.getRotationMatrix2D(center, -angle, 1)
landmark_ = np.asarray([
(rot_mat[0][0] * x + rot_mat[0][1] * y + rot_mat[0][2],
rot_mat[1][0] * x + rot_mat[1][1] * y + rot_mat[1][2])
for (x, y) in landmark
])
# print('landmark shape:', landmark_.shape)
return landmark_
assert (animojis_pred.shape[1:] == (70, 2)), str(animojis_pred.shape)
assert (len(boxes_pred.shape) == 2), str(boxes_pred.shape)
assert (len(landmarks_pred.shape) == 3), str(landmarks_pred.shape)
assert (len(animojis_pred.shape) == 3), str(animojis_pred.shape)
iou, _ = IoU(bbox, boxes_pred)
idx = np.argmax(iou)
if iou[idx] < 0.4:
max_iou.append(iou[idx])
error_detect_face.append(filepath)
continue
#######
result = ratate(boxes_pred[idx],
landmarks_pred[idx][0:2],
points=[animoji.reshape(70, 2)])
img_rotated, (animoji_rotated, ), _ = result
animoji_rotated = Animoji(animoji_rotated)
face_point = animoji_rotated.face
p1 = np.min(face_point, axis=0)
p2 = np.max(face_point, axis=0)
p3 = boxes_pred[idx][0:2]
p4 = boxes_pred[idx][2:4]
dx1, dy1 = p3 - p1
dx2, dy2 = p2 - p4
Lx, Ly = p4 - p3 + 1
def predict(self, im, boxes, landmarks):
img_rotated, (landmark_pred_rotated, ), (angle, center) = ratate(
boxes, landmarks, points=[landmarks], img=im)
h, w, c = img_rotated.shape
landmark_left_eye = landmark_pred_rotated[0]
landmark_right_eye = landmark_pred_rotated[1]
landmark_left_mouse = landmark_pred_rotated[3]
W = landmark_right_eye[0] - landmark_left_eye[0] + 1
H = landmark_left_mouse[1] - landmark_left_eye[1] + 1
dx1, dy1, dx2, dy2 = W * 0.8, H * 0.7, W * 0.8, H * 0.4
lx, ly = landmark_left_eye
lx1 = max(0, int(lx - dx1 + 0.5))
ly1 = max(0, int(ly - dy1 + 0.5))
lx2 = min(w - 1, int(lx + dx2 + 0.5))
ly2 = min(h - 1, int(ly + dy2 + 0.5))
Llx = lx2 - lx1 + 1
Lly = ly2 - ly1 + 1
rx, ry = landmark_right_eye
rx1 = max(0, int(rx - dx2 + 0.5))
ry1 = max(0, int(ry - dy1 + 0.5))
rx2 = min(w - 1, int(rx + dx1 + 0.5))
ry2 = min(h - 1, int(ry + dy2 + 0.5))
Lrx = rx2 - rx1 + 1
Lry = ry2 - ry1 + 1
left_im = img_rotated[ly1:ly2, lx1:lx2, :]
right_im = img_rotated[ry1:ry2, rx1:rx2, :]
right_im = cv2.flip(right_im, 1)
# cv2.imwrite('left.jpg', left_im)
# cv2.imwrite('right.jpg', right_im)
cropped_im1 = np.zeros((1, self.net_size, self.net_size, 3),
dtype=np.float32)
cropped_im1[0, :, :, :] = (cv2.resize(left_im,
(self.net_size, self.net_size)) -
127.5) / 128
cropped_im2 = np.zeros((1, self.net_size, self.net_size, 3),
dtype=np.float32)
cropped_im2[0, :, :, :] = (cv2.resize(right_im,
(self.net_size, self.net_size)) -
127.5) / 128
landmark1 = self.net.predict(cropped_im1)
landmark2 = self.net.predict(cropped_im2)
landmark1 = landmark1.reshape(-1, 2)
landmark2 = landmark2.reshape(-1, 2)
# print(landmark2.shape)
idx = [4, 3, 2, 1, 0, 8, 7, 6, 5, 10, 9, 11]
landmark2 = landmark2[idx, :]
landmark2[:, 0] = 1 - landmark2[:, 0]
lwh = np.asarray([[Llx, Lly]], dtype=np.float32)
lxy = np.asarray([[lx1, ly1]], dtype=np.float32)
rwh = np.asarray([[Lrx, Lry]], dtype=np.float32)
rxy = np.asarray([[rx1, ry1]], dtype=np.float32)
landmark1 = landmark1 * lwh + lxy
landmark2 = landmark2 * rwh + rxy
rot_mat = cv2.getRotationMatrix2D(center, -angle, 1)
landmark1_ = np.asarray([
(rot_mat[0][0] * x + rot_mat[0][1] * y + rot_mat[0][2],
rot_mat[1][0] * x + rot_mat[1][1] * y + rot_mat[1][2])
for (x, y) in landmark1
])
landmark2_ = np.asarray([
(rot_mat[0][0] * x + rot_mat[0][1] * y + rot_mat[0][2],
rot_mat[1][0] * x + rot_mat[1][1] * y + rot_mat[1][2])
for (x, y) in landmark2
])
return landmark1_, landmark2_
assert (animojis_pred.shape[1:] == (70, 2)), str(animojis_pred.shape)
assert (len(boxes_pred.shape) == 2), str(boxes_pred.shape)
assert (len(landmarks_pred.shape) == 3), str(landmarks_pred.shape)
assert (len(animojis_pred.shape) == 3), str(animojis_pred.shape)
iou = IoU(bbox, boxes_pred)
idx = np.argmax(iou)
if iou[idx] < 0.4:
max_iou.append(iou[idx])
error_detect_face.append(filepath)
continue
# img = cv2.imread(filepath)
result = ratate(boxes_pred[idx],
landmarks_pred[idx],
points=[animoji.reshape(70, 2), animojis_pred[idx]])
landmark_pred_rotated, img_rotated, (animoji_rotated,
animoji_pred_rotated) = result
center = np.asarray([[(boxes_pred[idx][0] + boxes_pred[idx][2]) / 2,
(boxes_pred[idx][1] + boxes_pred[idx][3]) / 2]],
dtype=np.float32)
# print(center.shape)
left_eye_point = animoji_rotated[left_eye_idx]
right_eye_point = animoji_rotated[right_eye_idx]
nose_point = animoji_rotated[nose_idx]
mouse_point = animoji_rotated[mouse_idx]
face_point = animoji_rotated[face_idx]
landmark_left_eye = landmark_pred_rotated[0]
iou = IoU(bbox, boxes_pred)
idx = np.argmax(iou)
if iou[idx] < 0.4:
# max_iou.append(iou[idx])
# error_detect_face.append(filepath)
continue
img = cv2.imread(filepath)
ratate_vector = landmarks_pred[idx][0:2]
out_points = [
landmarks_pred[idx],
animoji.reshape(70, 2), animojis_pred[idx]
]
result = ratate(boxes_pred[idx], ratate_vector, points=out_points, img=img)
img_rotated, (landmark_pred_rotated, animoji_rotated,
animoji_pred_rotated), _ = result
Animoji_rotated = Animoji(animoji_rotated)
Animoji_pred_rotated = Animoji(animoji_pred_rotated)
output_image_path = os.path.join(output_image, filename)
if not os.path.exists(os.path.dirname(output_image_path)):
os.makedirs(os.path.dirname(output_image_path))
cv2.imwrite(output_image_path, img_rotated)
imgh, imgw, imgc = img_rotated.shape
face_flipped_by_x = cv2.flip(img_rotated, 1)
name, ext = os.path.splitext(filename)
flipped_name = name + '_filpped' + ext
output_flipped_image_path = os.path.join(output_image, flipped_name)