def parse_mpiigaze_landmark_coords(person_id: int, day: int, img_n: int, eye_image_width=60, eye_image_height=36):
face_model = load_face_model()
person_id_str = str(person_id).zfill(2)
day_str = str(day).zfill(2)
img_n_str = str(img_n).zfill(4)
im = load_image_by_cv2(mpiigaze_path_wrapper(f"Data/Original/p{person_id_str}/day{day_str}/{img_n_str}.jpg"))
ann_path = mpiigaze_path_wrapper(f"Data/Original/p{person_id_str}/day{day_str}/annotation.txt")
annotation = load_annotation(ann_path)
camera_matrix = load_camera_matrix(path=f"Data/Original/p{person_id_str}/Calibration/Camera.mat")
im_height, im_width, _ = im.shape
headpose_hr = np.reshape(annotation[img_n - 1, 29:32], (1, -1))
headpose_ht = np.reshape(annotation[img_n - 1, 32:35], (1, -1))
h_r, _ = cv2.Rodrigues(headpose_hr)
fc = np.dot(h_r, face_model)
fc = fc + np.reshape(headpose_ht, (-1, 1))
gaze_target = annotation[img_n - 1, 26:29]
gaze_target = np.reshape(gaze_target, (-1, 1))
right_eye_center = 0.5 * (fc[:, 0] + fc[:, 1])
left_eye_center = 0.5 * (fc[:, 2] + fc[:, 3])
right_eye = get_img_gaze_headpose_per_eye(im, right_eye_center, h_r, gaze_target,
eye_image_width, eye_image_height, camera_matrix)
left_eye = get_img_gaze_headpose_per_eye(im, left_eye_center, h_r, gaze_target,
eye_image_width, eye_image_height, camera_matrix)
landmarks = np.reshape(annotation[img_n - 1, 0:24], (1, -1))
landmarks = norm_landmarks(landmarks, height=im_height, width=im_width)
coordinates = np.array([annotation[img_n - 1, 25], annotation[img_n - 1, 24]])
return right_eye, left_eye, landmarks, coordinates
def get_all_images_ids_for_day(person_id, day_id):
person_id_str = f"p{str(person_id).zfill(2)}"
day = f"day{str(day_id).zfill(2)}"
im_filenames = get_all_jpg_files(
mpiigaze_path_wrapper(f"Data/Original/{person_id_str}/{day}/"))
ids = [int(filename.replace(".jpg", "")) for filename in im_filenames]
return ids
def get_all_days_ids(person_id):
"""
Function retrieves days ids from strings like 'day{xx}'.
:days_list_str: list containing days as str: day{xx} where xx is id
"""
person_id_str = f"p{str(person_id).zfill(2)}"
days_list_str = get_all_days(
path=mpiigaze_path_wrapper(f"Data/Original/{person_id_str}/"))
ids = [int(day.replace("day", "")) for day in days_list_str]
return ids
def load_screen_size(path, type="pixel", face=False):
"""
:param path:
:param type: 'pixel' or 'mm' (millimeter)
:return:
"""
if face:
path = mpii_face_gaze_path_wrapper(path)
else:
path = mpiigaze_path_wrapper(path)
matdata = scipy.io.loadmat(path, struct_as_record=False, squeeze_me=True)
h, w = matdata[f"height_{type}"], matdata[f"width_{type}"]
return (h, w)
def get_mpiigaze_data():
df_untrusted = pd.DataFrame(columns=["person_id", "day", "counter"])
image_counter = dict()
x_coords = list()
y_coords = list()
for person_id in range(15):
person_id_str = str(person_id).zfill(2)
screen_size = load_screen_size(
path=f"Data/Original/p{person_id_str}/Calibration/screenSize.mat")
days_ids = get_all_days_ids(person_id)
for day_id in days_ids:
day_str = str(day_id).zfill(2)
annotation = np.loadtxt(
mpiigaze_path_wrapper(
f"Data/Original/p{person_id_str}/day{day_str}/annotation.txt"
))
if len(annotation.shape) == 1:
annotation = annotation.reshape((1, -1))
im_ids = get_all_images_ids_for_day(person_id, day_id)
for im_id in im_ids:
coordinates = np.array(
[annotation[im_id - 1, 25], annotation[im_id - 1, 24]])
if float(coordinates[0]) / screen_size[0] > 1.0 or float(
coordinates[1]) / screen_size[1] > 1.0:
df_untrusted = df_untrusted.append([{
"person_id": person_id_str,
"day": day_str,
"counter": 1
}],
ignore_index=True)
continue
if person_id in image_counter:
image_counter[person_id] += 1
else:
image_counter[person_id] = 1
x_coords.append(float(coordinates[0]) / screen_size[0])
y_coords.append(float(coordinates[1]) / screen_size[1])
return image_counter, x_coords, y_coords, df_untrusted
def create_dataset_mpiigaze_processed_both_rgb_full_transformation(
dataset_name):
create_dirs(dataset_name)
face_model = load_face_model()
eye_image_width = 60
eye_image_height = 36
for person_id in range(15):
person_id_str = str(person_id).zfill(2)
print(f"--------------\nperson_id_str: {person_id_str}")
camera_matrix = load_camera_matrix(
path=f"Data/Original/p{person_id_str}/Calibration/Camera.mat")
screen_size = load_screen_size(
path=f"Data/Original/p{person_id_str}/Calibration/screenSize.mat")
print(screen_size)
day_dirs = get_all_days(
path=mpiigaze_path_wrapper(f"Data/Original/p{person_id_str}/"))
for day in day_dirs:
left_eyes = list()
right_eyes = list()
headposes = list()
gazes = list()
coordinates = list()
print(day)
ann_path = mpiigaze_path_wrapper(
f"Data/Original/p{person_id_str}/{day}/annotation.txt")
annotation = load_annotation(ann_path)
im_filenames = get_all_jpg_files(
mpiigaze_path_wrapper(
f"Data/Original/p{person_id_str}/{day}/"))
for i in tqdm(range(len(im_filenames))):
im_file = im_filenames[i]
im = load_image_by_cv2(
mpiigaze_path_wrapper(
f"Data/Original/p{person_id_str}/{day}/{im_file}"))
headpose_hr = np.reshape(annotation[i, 29:32], (1, -1))
headpose_ht = np.reshape(annotation[i, 32:35], (1, -1))
h_r, _ = cv2.Rodrigues(headpose_hr)
fc = np.dot(h_r, face_model)
fc = fc + np.reshape(headpose_ht, (-1, 1))
gaze_target = annotation[i, 26:29]
gaze_target = np.reshape(gaze_target, (-1, 1))
right_eye_center = 0.5 * (fc[:, 0] + fc[:, 1])
left_eye_center = 0.5 * (fc[:, 2] + fc[:, 3])
right_eye_img, right_eye_headpose, right_eye_gaze = mpii_gaze_normalize_image(
im, right_eye_center, h_r, gaze_target,
(eye_image_width, eye_image_height), camera_matrix)
left_eye_img, left_eye_headpose, left_eye_gaze = mpii_gaze_normalize_image(
im, left_eye_center, h_r, gaze_target,
(eye_image_width, eye_image_height), camera_matrix)
right_eyes.append(right_eye_img)
left_eyes.append(left_eye_img)
headpose_angles = list()
headpose_angles.extend(
count_headpose_angles(right_eye_headpose /
np.linalg.norm(right_eye_headpose)))
headpose_angles.extend(
count_headpose_angles(left_eye_headpose /
np.linalg.norm(left_eye_headpose)))
headpose_angles.extend(
count_headpose_angles(headpose_hr /
np.linalg.norm(headpose_hr)))
headposes.append(np.array(headpose_angles))
gazes.append(
count_gaze_angles(gaze_target /
np.linalg.norm(gaze_target)))
coordinates.append([annotation[i, 25], annotation[i, 24]])
coordinates = norm_coords(coordinates, screen_size)
# save_coords(person_id_str, day, coordinates)
save_dataset_mpiigaze_processed_both_rgb(person_id_str,
day,
right_eyes,
left_eyes,
headposes,
gazes,
coordinates,
dataset_name=dataset_name)
def load_camera_matrix(path):
path = mpiigaze_path_wrapper(path)
matdata = scipy.io.loadmat(path, struct_as_record=False, squeeze_me=True)
return matdata["cameraMatrix"]
def create_dataset_mpiigaze_processed_both_rgb(dataset_name, headpose_type):
"""
This function creates dataset with following record structure:
(right_eye_rgb_img, left_eye_rgb_img, gaze_target_theta, gaze_target_phi, norm_x_coordinate, norm_y_coordinate).
Records are saved to pickles. One pickle for one person_id_str.
:return:
"""
create_dirs(dataset_name)
face_model = load_face_model()
eye_image_width = 60
eye_image_height = 36
for person_id in range(15):
person_id_str = str(person_id).zfill(2)
print(f"--------------\nperson_id_str: {person_id_str}")
camera_matrix = load_camera_matrix(
path=f"Data/Original/p{person_id_str}/Calibration/Camera.mat")
screen_size = load_screen_size(
path=f"Data/Original/p{person_id_str}/Calibration/screenSize.mat")
print(screen_size)
day_dirs = get_all_days(
path=mpiigaze_path_wrapper(f"Data/Original/p{person_id_str}/"))
for day in day_dirs:
left_eyes = list()
right_eyes = list()
headposes = list()
gazes = list()
coordinates = list()
print(day)
ann_path = mpiigaze_path_wrapper(
f"Data/Original/p{person_id_str}/{day}/annotation.txt")
annotation = load_annotation(ann_path)
im_filenames = get_all_jpg_files(
mpiigaze_path_wrapper(
f"Data/Original/p{person_id_str}/{day}/"))
for i in tqdm(range(len(im_filenames))):
im_file = im_filenames[i]
im = load_image_by_cv2(
mpiigaze_path_wrapper(
f"Data/Original/p{person_id_str}/{day}/{im_file}"))
headpose_hr = np.reshape(annotation[i, 29:32], (1, -1))
headpose_ht = np.reshape(annotation[i, 32:35], (1, -1))
h_r, _ = cv2.Rodrigues(headpose_hr)
fc = np.dot(h_r, face_model)
fc = fc + np.reshape(headpose_ht, (-1, 1))
gaze_target = annotation[i, 26:29]
gaze_target = np.reshape(gaze_target, (-1, 1))
right_eye_center = 0.5 * (fc[:, 0] + fc[:, 1])
left_eye_center = 0.5 * (fc[:, 2] + fc[:, 3])
right_eye_img = cut_eye(im, right_eye_center, h_r,
(eye_image_width, eye_image_height),
camera_matrix)
left_eye_img = cut_eye(im, left_eye_center, h_r,
(eye_image_width, eye_image_height),
camera_matrix)
right_eyes.append(right_eye_img)
left_eyes.append(left_eye_img)
if headpose_type == "2_3_dim_vectors":
headposes.append(
np.concatenate((headpose_hr, headpose_ht),
axis=1).squeeze())
elif headpose_type == "2_angles":
headposes.append(
count_headpose_angles(headpose_hr /
np.linalg.norm(headpose_hr)))
gazes.append(
count_gaze_angles(gaze_target /
np.linalg.norm(gaze_target)))
coordinates.append([annotation[i, 25], annotation[i, 24]])
coordinates = norm_coords(coordinates, screen_size)
# save_coords(person_id_str, day, coordinates)
save_dataset_mpiigaze_processed_both_rgb(person_id_str,
day,
right_eyes,
left_eyes,
headposes,
gazes,
coordinates,
dataset_name=dataset_name)
def load_face_model():
path = mpiigaze_path_wrapper("6 points-based face model.mat")
matdata = scipy.io.loadmat(path, struct_as_record=False, squeeze_me=True)
return matdata["model"]
# Take only first face
if len(faces) > 1:
faces = faces[0].reshape(1, -1)
elif len(faces) == 0: # No face detected -> assume that face is on the full image
return None
# detect landmarks
landmarks = landmarks_detector.detect(image_gray, faces)
visualize_landmarks_mpii_gaze_format(landmarks[:, EYES_LANDMARKS, :], image, numbers=False)
if __name__ == "__main__":
"""
Script prints all landmarks from MPII Gaze dataset on photo, to know which landmarks are they.
"""
person_id_str = "14"
day = "day04"
im_file = "0008.jpg"
im = load_image_by_cv2(mpii_face_gaze_path_wrapper(f"p{person_id_str}/{day}/{im_file}"))
annotation = np.loadtxt(mpiigaze_path_wrapper(f"Data/Original/p{person_id_str}/{day}/annotation.txt"))
landmarks = np.reshape(annotation[8 - 1, 0:24], (1, -1, 2))
landmarks = landmarks.astype(np.int)
visualize_landmarks_mpii_gaze_format(landmarks, im, numbers=True)
# haarcascade_lbf(im)
im = Image.fromarray(im)
im.save(os.path.join(FOR_THESIS_DIR, "eye_landmarks.png"))