def main(args):
det = hdface_detector(use_cuda=False)
checkpoint = torch.load(args.model_path)
plfd_backbone = PFLDInference().cuda()
plfd_backbone.load_state_dict(checkpoint)
plfd_backbone.eval()
plfd_backbone = plfd_backbone.cuda()
transform = transforms.Compose([transforms.ToTensor()])
root = args.images_path
path_list = glob.glob(os.path.join(root, "*.jpg"))
# cap = cv2.VideoCapture("")
for img_path in path_list:
img = cv2.imread(img_path)
height, width = img.shape[:2]
img_det = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
result = det.detect_face(img_det)
for i in range(len(result)):
box = result[i]['box']
cls = result[i]['cls']
pts = result[i]['pts']
x1, y1, x2, y2 = box
cv2.rectangle(img, (x1, y1), (x2, y2), (255, 255, 25))
w = x2 - x1 + 1
h = y2 - y1 + 1
size_w = int(max([w, h]) * 0.9)
size_h = int(max([w, h]) * 0.9)
cx = x1 + w // 2
cy = y1 + h // 2
x1 = cx - size_w // 2
x2 = x1 + size_w
y1 = cy - int(size_h * 0.4)
y2 = y1 + size_h
left = 0
top = 0
bottom = 0
right = 0
if x1 < 0:
left = -x1
if y1 < 0:
top = -y1
if x2 >= width:
right = x2 - width
if y2 >= height:
bottom = y2 - height
x1 = max(0, x1)
y1 = max(0, y1)
x2 = min(width, x2)
y2 = min(height, y2)
cropped = img[y1:y2, x1:x2]
print(top, bottom, left, right)
cropped = cv2.copyMakeBorder(cropped, top, bottom, left, right,
cv2.BORDER_CONSTANT, 0)
cropped = cv2.resize(cropped, (112, 112))
input = cv2.resize(cropped, (112, 112))
input = cv2.cvtColor(input, cv2.COLOR_BGR2RGB)
input = transform(input).unsqueeze(0).cuda()
pose, landmarks = plfd_backbone(input)
poses = pose.cpu().detach().numpy()[0] * 180 / np.pi
pre_landmark = landmarks[0]
pre_landmark = pre_landmark.cpu().detach().numpy().reshape(
-1, 2) * [size_w, size_h]
cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0))
for (x, y) in pre_landmark.astype(np.int32):
cv2.circle(img, (x1 - left + x, y1 - bottom + y), 1,
(255, 255, 0), 1)
plot_pose_cube(img,
poses[0],
poses[1],
poses[2],
tdx=pts['nose'][0],
tdy=pts['nose'][1],
size=(x2 - x1) // 2)
cv2.imshow('0', img)
cv2.waitKey(0)
def validate(wlfw_val_dataloader, plfd_backbone, args):
plfd_backbone.eval()
with torch.no_grad():
losses_NME = []
losses_ION = []
for img, landmark_gt, attribute_gt, euler_angle_gt, type_flag in wlfw_val_dataloader:
img.requires_grad = False
img = img.cuda(non_blocking=True)
attribute_gt.requires_grad = False
attribute_gt = attribute_gt.cuda(non_blocking=True)
landmark_gt.requires_grad = False
landmark_gt = landmark_gt.cuda(non_blocking=True)
euler_angle_gt.requires_grad = False
euler_angle_gt = euler_angle_gt.cuda(non_blocking=True)
type_flag.requires_grad = False
type_flag = type_flag.cuda()
plfd_backbone = plfd_backbone.cuda()
pose, landmarks = plfd_backbone(img)
if args.is_show:
landms_tmp = landmarks.cpu().numpy()
landms_tmp = landms_tmp.reshape(landms_tmp.shape[0], -1, 2)
for i in range(landms_tmp.shape[0]):
# show result
show_img = np.array(
np.transpose(img[i].cpu().numpy(), (1, 2, 0)))
show_img = (show_img * 255).astype(np.uint8)
np.clip(show_img, 0, 255)
draw = show_img.copy()
yaw = pose[i][0] * 180 / np.pi
pitch = pose[i][1] * 180 / np.pi
roll = pose[i][2] * 180 / np.pi
pre_landmark = landms_tmp[i] * [112, 112]
for (x, y) in pre_landmark.astype(np.int8):
cv2.circle(draw, (int(x), int(y)), 1, (0, 255, 0), 1)
draw = plot_pose_cube(draw,
yaw,
pitch,
roll,
size=draw.shape[0] // 2)
cv2.imshow("check", draw)
cv2.waitKey(0)
landms_const = torch.tensor(-2).cuda()
pose_const = torch.tensor(-1).cuda()
pos1 = type_flag == landms_const
landmarks = landmarks[pos1]
landmark_gt = landmark_gt[pos1]
if landmarks.shape[0] > 0:
loss = torch.mean(
torch.sqrt(torch.sum((landmark_gt - landmarks)**2, dim=1)))
landmarks = landmarks.cpu().numpy()
landmarks = landmarks.reshape(landmarks.shape[0], -1, 2)
landmark_gt = landmark_gt.reshape(landmark_gt.shape[0], -1,
2).cpu().numpy()
error_diff = np.sum(np.sqrt(
np.sum((landmark_gt - landmarks)**2, axis=2)),
axis=1)
interocular_distance = np.sqrt(
np.sum((landmarks[:, 60, :] - landmarks[:, 72, :])**2,
axis=1))
# interpupil_distance = np.sqrt(np.sum((landmarks[:, 60, :] - landmarks[:, 72, :]) ** 2, axis=1))
error_norm = np.mean(error_diff / interocular_distance)
losses_NME.append(loss.cpu().numpy())
losses_ION.append(error_norm)
print("NME", np.mean(losses_NME))
print("ION", np.mean(losses_ION))
def main():
args = get_args()
img_size = args.img_size
pts_68_index = pts_68_index_table()
output_db_root = os.path.join(args.output, args.db)
if not os.path.exists(output_db_root):
os.makedirs(output_db_root)
onlyfiles_mat = [
f for f in listdir(join(args.root, args.db))
if isfile(join(args.root, args.db, f))
and join(args.root, args.db, f).endswith('.mat')
]
onlyfiles_jpg = [
f for f in listdir(join(args.root, args.db))
if isfile(join(args.root, args.db, f))
and join(args.root, args.db, f).endswith('.jpg')
]
# AFW_134212_1_0_pts
onlyfiles_mat.sort()
onlyfiles_jpg.sort()
print(len(onlyfiles_jpg))
print(len(onlyfiles_mat))
out_imgs = []
out_poses = []
with open(os.path.join(args.output, args.db + '_label.txt'), 'w') as fw:
for i in tqdm(range(len(onlyfiles_jpg))):
img_name = onlyfiles_jpg[i]
mat_name = onlyfiles_mat[i]
save_path = args.db + '/' + img_name
# print(img_name)
# print(lds_name)
# print(mat_name)
img_name_split = img_name.split('.')
mat_name_split = mat_name.split('.')
if img_name_split[0] != mat_name_split[0]:
print('Mismatched!')
sys.exit()
mat_contents = sio.loadmat(
os.path.join(args.root, args.db, mat_name))
img = cv2.imread(os.path.join(args.root, args.db, img_name))
pose_para = mat_contents['Pose_Para'][0]
pt2d = mat_contents['pt2d']
pt2d_x = pt2d[0, :]
pt2d_y = pt2d[1, :]
# I found negative value in AFLW2000. It need to be removed.
pt2d_idx = pt2d_x > 0.0
pt2d_idy = pt2d_y > 0.0
pt2d_id = pt2d_idx
if sum(pt2d_idx) > sum(pt2d_idy):
pt2d_id = pt2d_idy
pt2d_x = pt2d_x[pt2d_id]
pt2d_y = pt2d_y[pt2d_id]
img_h = img.shape[0]
img_w = img.shape[1]
# Crop the face loosely
x_min = int(min(pt2d_x))
x_max = int(max(pt2d_x))
y_min = int(min(pt2d_y))
y_max = int(max(pt2d_y))
h = y_max - y_min
w = x_max - x_min
# ad = 0.4
ad = random.uniform(0.12, 0.3)
x_min = max(int(x_min - ad * w), 0)
x_max = min(int(x_max + ad * w), img_w - 1)
y_min = max(int(y_min - ad * h), 0)
y_max = min(int(y_max + ad * h), img_h - 1)
img = img[y_min:y_max, x_min:x_max]
# Checking the cropped image
w_factor = img_size / float(img.shape[1])
h_factor = img_size / float(img.shape[0])
img = cv2.resize(img, (img_size, img_size))
save_img = img.copy()
landmarks98 = np.array([-256.0 for i in range(98 * 2)])
for k in range(pt2d_x.shape[0]):
center = (int((pt2d_x[k] - x_min) * w_factor),
int((pt2d_y[k] - y_min) * h_factor))
cv2.circle(img, center, 1, (255, 255, 0), 1)
pitch = pose_para[0] * 180 / np.pi
yaw = pose_para[1] * 180 / np.pi
roll = pose_para[2] * 180 / np.pi
euler_angles = np.array([yaw, pitch, roll])
plot_pose_cube(img, yaw, pitch, roll, tdx=50, tdy=50, size=50)
landmark_str = ' '.join(
list(map(str,
landmarks98.reshape(-1).tolist())))
euler_angles_str = ' '.join(
list(map(str,
euler_angles.reshape(-1).tolist())))
label = '{} {} 0 0 0 0 0 0 {}\n'.format(save_path, landmark_str,
euler_angles_str)
fw.write(label)
cv2.imwrite(os.path.join(output_db_root, img_name), save_img)
if args.is_show:
cv2.imshow('check', img)
cv2.waitKey(0)
def validate(wlfw_val_dataloader, plfd_backbone):
plfd_backbone.eval()
with torch.no_grad():
losses = []
losses_ION = []
for img, landmark_gt, attribute_gt, euler_angle_gt in wlfw_val_dataloader:
img.requires_grad = False
img = img.cuda(non_blocking=True)
attribute_gt.requires_grad = False
attribute_gt = attribute_gt.cuda(non_blocking=True)
landmark_gt.requires_grad = False
landmark_gt = landmark_gt.cuda(non_blocking=True)
euler_angle_gt.requires_grad = False
euler_angle_gt = euler_angle_gt.cuda(non_blocking=True)
plfd_backbone = plfd_backbone.cuda()
pose, landmarks = plfd_backbone(img)
loss = torch.mean(
torch.sqrt(torch.sum((landmark_gt - landmarks)**2, dim=1))
)
landmarks = landmarks.cpu().numpy()
landmarks = landmarks.reshape(landmarks.shape[0], -1, 2)
landmark_gt = landmark_gt.reshape(landmark_gt.shape[0], -1, 2).cpu().numpy()
error_diff = np.sum(np.sqrt(np.sum((landmark_gt - landmarks) ** 2, axis=2)), axis=1)
interocular_distance = np.sqrt(np.sum((landmarks[:, 60, :] - landmarks[:,72, :]) ** 2, axis=1))
# interpupil_distance = np.sqrt(np.sum((landmarks[:, 60, :] - landmarks[:, 72, :]) ** 2, axis=1))
error_norm = np.mean(error_diff / interocular_distance)
# show result
show_img = np.array(np.transpose(img[0].cpu().numpy(), (1, 2, 0)))
show_img = (show_img * 255).astype(np.uint8)
np.clip(show_img, 0, 255)
pre_landmark = landmarks[0] * [112, 112]
# cv2.imwrite("xxx.jpg", show_img)
# img_clone = cv2.imread("xxx.jpg")
draw = show_img.copy()
yaw = pose[0][0] * 180 / np.pi
pitch = pose[0][1] * 180 / np.pi
roll = pose[0][2] * 180 / np.pi
for (x, y) in pre_landmark.astype(np.int8):
# print("x:{0:}, y:{1:}".format(x, y))
cv2.circle(draw, (int(x), int(y)), 1, (0,255,0), 1)
draw = plot_pose_cube(draw, yaw, pitch, roll, size=draw.shape[0] // 2)
# cv2.imshow("xx.jpg", draw)
# cv2.waitKey(0)
losses.append(loss.cpu().numpy())
losses_ION.append(error_norm)
print("NME", np.mean(losses))
print("ION", np.mean(losses_ION))
def main():
args = get_args()
img_size = args.img_size
pts_68_index = pts_68_index_table()
output_db_root = os.path.join(args.output, args.db)
if not os.path.exists(output_db_root):
os.makedirs(output_db_root)
onlyfiles_mat = [
f for f in listdir(join(args.root, args.db))
if isfile(join(args.root, args.db, f))
and join(args.root, args.db, f).endswith('.mat')
]
onlyfiles_jpg = [
f for f in listdir(join(args.root, args.db))
if isfile(join(args.root, args.db, f))
and join(args.root, args.db, f).endswith('.jpg')
]
# AFW_134212_1_0_pts
onlyfiles_mat.sort()
onlyfiles_jpg.sort()
print(len(onlyfiles_jpg))
print(len(onlyfiles_mat))
out_imgs = []
out_poses = []
with open(os.path.join(args.output, args.db + '_label.txt'), 'w') as fw:
for i in tqdm(range(len(onlyfiles_jpg))):
img_name = onlyfiles_jpg[i]
lds_name = img_name.split('/')[-1].replace(".jpg", "_pts.mat")
mat_name = onlyfiles_mat[i]
# print(img_name)
# print(lds_name)
# print(mat_name)
img_name_split = img_name.split('.')
mat_name_split = mat_name.split('.')
if img_name_split[0] != mat_name_split[0]:
print('Mismatched!')
sys.exit()
mat_contents = sio.loadmat(
os.path.join(args.root, args.db, mat_name))
lds_contents = sio.loadmat(
os.path.join(args.root, 'landmarks', args.db, lds_name))
img = cv2.imread(os.path.join(args.root, args.db, img_name))
pose_para = mat_contents['Pose_Para'][0]
pt2d = mat_contents['pt2d']
pt2d_real = lds_contents['pts_2d']
pt2d_x = pt2d_real[:, 0]
pt2d_y = pt2d_real[:, 1]
img_h = img.shape[0]
img_w = img.shape[1]
# Crop the face loosely
x_min = int(min(pt2d_x))
x_max = int(max(pt2d_x))
y_min = int(min(pt2d_y))
y_max = int(max(pt2d_y))
h = y_max - y_min
w = x_max - x_min
# ad = 0.4
ad = random.uniform(0.05, 0.2)
x_min = max(int(x_min - ad * w), 0)
x_max = min(int(x_max + ad * w), img_w - 1)
y_min = max(int(y_min - ad * h), 0)
y_max = min(int(y_max + ad * h), img_h - 1)
img = img[y_min:y_max, x_min:x_max]
# Checking the cropped image
w_factor = img_size / float(img.shape[1])
h_factor = img_size / float(img.shape[0])
img = cv2.resize(img, (img_size, img_size))
save_img = img.copy()
landmarks98 = np.array([-256.0 for i in range(98 * 2)])
for k in range(pt2d_x.shape[0]):
center = (int((pt2d_x[k] - x_min) * w_factor),
int((pt2d_y[k] - y_min) * h_factor))
cv2.circle(img, center, 1, (255, 255, 0), 1)
points = (center[0] / 112.0, center[1] / 112.0)
landmarks98[pts_68_index[str(k)] * 2 + 0] = points[0]
landmarks98[pts_68_index[str(k)] * 2 + 1] = points[1]
# cv2.putText(img, str(pts_68_index[str(k)]), center, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.8, (0,255,0), 1)
pitch = pose_para[0] * 180 / np.pi
yaw = pose_para[1] * 180 / np.pi
roll = pose_para[2] * 180 / np.pi
euler_angles = np.array([yaw, pitch, roll])
plot_pose_cube(img, yaw, pitch, roll, tdx=100, tdy=100, size=50)
landmark_str = ' '.join(
list(map(str,
landmarks98.reshape(-1).tolist())))
euler_angles_str = ' '.join(
list(map(str,
euler_angles.reshape(-1).tolist())))
img_name = img_name.replace(' ', '')
save_path = args.db + '/' + img_name
label = '{} {} 0 0 0 0 0 0 {}\n'.format(save_path, landmark_str,
euler_angles_str)
fw.write(label)
cv2.imwrite(os.path.join(output_db_root, img_name), save_img)
if args.is_show:
cv2.imshow('check', img)
cv2.waitKey(0)