class DetectGaze:
def __init__(self, h5_path, pts_path="../../faces/data"):
self.curlandmark_file_path = ""
self.detector = DlibFaceDetector(pts_path)
self.model = KerasELG()
self.model.net.load_weights(h5_path)
def draw_pupil(self, lms):
actual_pos = []
# 0-7 eye corner
# 8-15 iris
# 16 pupil
for i, lm in enumerate(np.squeeze(lms)):
#print(lm)
# y 就是x，x就是y
y, x = int(lm[0] * 3), int(lm[1] * 3)
actual_pos.append([y, x])
return actual_pos
# xy
def recalPos(self, pos, imagew, imageh):
_recalPos = []
for i, _pos in enumerate(pos):
_recalPos.append(
[int(_pos[0] / 180 * imagew),
int(_pos[1] / 108 * imageh)])
return _recalPos
def detect_iris(self, image_path):
input_img = cv2.imread(image_path)[..., ::-1]
self.detector.curlandmark_file_path = self.curlandmark_file_path
left_eye_center, right_eye_center = self.detector.detect_face(
image_path)
# print(left_eye_center)
# print(right_eye_center)
left_eye_center = [int(left_eye_center[0]), int(left_eye_center[1])]
right_eye_center = [int(right_eye_center[0]), int(right_eye_center[1])]
# left_eye_xy = np.array([lms[6], lms[1]])
# right_eye_xy = np.array([lms[5], lms[0]])
left_eye_xy = np.array([left_eye_center[0], left_eye_center[1]])
right_eye_xy = np.array([right_eye_center[0], right_eye_center[1]])
dist_eyes = np.linalg.norm(left_eye_xy - right_eye_xy)
# bounding box
eye_bbox_w = (dist_eyes / 1.25)
eye_bbox_h = (eye_bbox_w * 0.6)
# print(dist_eyes)
# print(eye_bbox_w)
# print(eye_bbox_h)
# draw = input_img.copy()
###
# ——————————————
# | |
# | |
# | * |
# | |
# |_____________|
###
# left_eye_xy.astype(int)
# right_eye_xy.astype(int)
left_eye_im = input_img[int(left_eye_xy[1] -
eye_bbox_h // 2):int(left_eye_xy[1] +
eye_bbox_h // 2),
int(left_eye_xy[0] -
eye_bbox_w // 2):int(left_eye_xy[0] +
eye_bbox_w // 2), :]
#left_eye_im = left_eye_im[:,::-1,:] # No need for flipping left eye for iris detection
right_eye_im = input_img[int(right_eye_xy[1] -
eye_bbox_h // 2):int(right_eye_xy[1] +
eye_bbox_h // 2),
int(right_eye_xy[0] -
eye_bbox_w // 2):int(right_eye_xy[0] +
eye_bbox_w // 2), :]
# draw = input_img.copy()
# draw = cv2.circle(draw, (left_eye_center[0], left_eye_center[1]), 5, (0,0,255), -1)
# draw = cv2.circle(draw, (right_eye_center[0], right_eye_center[1]), 5, (0,255,0), -1)
# draw = cv2.circle(draw, (int(left_eye_xy[0] - eye_bbox_w//2), int(left_eye_xy[1]- eye_bbox_h//2)), 5, (255,255,255), -1)
# draw = cv2.circle(draw, (int(left_eye_xy[0] + eye_bbox_w//2), int(left_eye_xy[1]- eye_bbox_h//2)), 5, (255,255,255), -1)
# draw = cv2.circle(draw, (int(left_eye_xy[0] - eye_bbox_w//2), int(left_eye_xy[1]+ eye_bbox_h//2)), 5, (255,255,255), -1)
# draw = cv2.circle(draw, (int(left_eye_xy[0] + eye_bbox_w//2), int(left_eye_xy[1]+ eye_bbox_h//2)), 5, (255,255,255), -1)
# draw = cv2.circle(draw, (int(right_eye_xy[0] - eye_bbox_w//2), int(right_eye_xy[1]- eye_bbox_h//2)), 5, (255,255,255), -1)
# draw = cv2.circle(draw, (int(right_eye_xy[0] + eye_bbox_w//2), int(right_eye_xy[1]- eye_bbox_h//2)), 5, (255,255,255), -1)
# draw = cv2.circle(draw, (int(right_eye_xy[0] - eye_bbox_w//2), int(right_eye_xy[1]+ eye_bbox_h//2)), 5, (255,255,255), -1)
# draw = cv2.circle(draw, (int(right_eye_xy[0] + eye_bbox_w//2), int(right_eye_xy[1]+ eye_bbox_h//2)), 5, (255,255,255), -1)
# draw = cv2.resize(draw,(draw.shape[1] // 2,draw.shape[0] // 2))
# cv2.imshow("image",draw)
# cv2.waitKey()
# draw = right_eye_im.copy()
# cv2.imshow("image",draw)
# cv2.waitKey()
# draw = left_eye_im.copy()
# cv2.imshow("image",draw)
# cv2.waitKey()
inp_left = cv2.cvtColor(left_eye_im, cv2.COLOR_RGB2GRAY)
inp_left = cv2.equalizeHist(inp_left)
# (180,108) ： 宽 and 高
inp_left = cv2.resize(inp_left, (180, 108))[np.newaxis, ...,
np.newaxis]
inp_right = cv2.cvtColor(right_eye_im, cv2.COLOR_RGB2GRAY)
inp_right = cv2.equalizeHist(inp_right)
inp_right = cv2.resize(inp_right, (180, 108))[np.newaxis, ...,
np.newaxis]
input_array = np.concatenate([inp_left, inp_right], axis=0)
pred_left, pred_right = self.model.net.predict(input_array / 255 * 2 -
1)
lms_left = self.model._calculate_landmarks(pred_left)
lms_right = self.model._calculate_landmarks(pred_right)
# 拼接
input_array = np.concatenate([inp_left, inp_right], axis=0)
pred_left, pred_right = self.model.net.predict(input_array / 255 * 2 -
1)
# g and b iris and pupil
lms_left = self.model._calculate_landmarks(pred_left)
lms_right = self.model._calculate_landmarks(pred_right)
lms_left = self.model._calculate_landmarks(pred_left)
actual_pos_left = self.draw_pupil(lms_left)
# x,y
actual_pos_left = self.recalPos(actual_pos_left, eye_bbox_w,
eye_bbox_h)
lms_right = self.model._calculate_landmarks(pred_right)
actual_pos_right = self.draw_pupil(lms_right)
actual_pos_right = self.recalPos(actual_pos_right, eye_bbox_w,
eye_bbox_h)
left_eye_im_leftTop = [
int(left_eye_xy[0] - eye_bbox_w // 2),
int(left_eye_xy[1] - eye_bbox_h // 2)
]
right_eye_im_leftTop = [
int(right_eye_xy[0] - eye_bbox_w // 2),
int(right_eye_xy[1] - eye_bbox_h // 2)
]
iris_lm_right = []
iris_lm_left = []
# iris landmark index [8:16]
for lm in actual_pos_left:
iris_lm_left.append([
int(left_eye_im_leftTop[0] + lm[0]),
int(left_eye_im_leftTop[1] + lm[1])
])
# print(lm)
for lm in actual_pos_right:
iris_lm_right.append([
int(right_eye_im_leftTop[0] + lm[0]),
int(right_eye_im_leftTop[1] + lm[1])
])
# draw = input_img.copy()
# draw = cv2.circle(draw, (left_eye_center[0], left_eye_center[1]), 5, (0,0,255), -1)
# draw = cv2.circle(draw, (right_eye_center[0], right_eye_center[1]), 5, (0,255,0), -1)
# draw = cv2.circle(draw, (left_eye_im_leftTop[0],left_eye_im_leftTop[1]), 5, (255,255,0), -1)
# draw = cv2.circle(draw, (right_eye_im_leftTop[0],right_eye_im_leftTop[1]), 5, (255,255,0), -1)
# draw = cv2.circle(draw, (iris_lm_left[0][0],iris_lm_left[0][1]), 5, (0,0,255), -1)
# # draw = cv2.circle(draw, (iris_lm_right[0][0],iris_lm_right[0][1]), 5, (0,0,255), -1)
# draw = cv2.resize(draw,(draw.shape[1] // 3,draw.shape[0] // 3))
# cv2.imshow("image",draw)
# cv2.waitKey()
# only iris landmark
return iris_lm_left, iris_lm_right
def __init__(self, h5_path, pts_path="../../faces/data"):
self.curlandmark_file_path = ""
self.detector = DlibFaceDetector(pts_path)
self.model = KerasELG()
self.model.net.load_weights(h5_path)
from detector.face_detector import MTCNNFaceDetector
from models.elg_keras import KerasELG
from keras import backend as K
import numpy as np
import cv2
import keras2onnx
from matplotlib import pyplot as plt
import onnxruntime
model = KerasELG()
model.net.load_weights("./elg_weights/elg_keras.h5")
onnx_model = keras2onnx.convert_keras(model.net, model.net.name)
temp_model_file = './model.onnx'
keras2onnx.save_model(onnx_model, temp_model_file)
# xy
def recalPos(pos, imagew, imageh):
_recalPos = []
for i, _pos in enumerate(pos):
_recalPos.append(
[int(_pos[0] / 180 * imagew),
int(_pos[1] / 108 * imageh)])
return _recalPos
mtcnn_weights_dir = "./mtcnn_weights/"
fd = MTCNNFaceDetector(sess=K.get_session(), model_path=mtcnn_weights_dir)
model = KerasELG()
model.net.load_weights("./elg_weights/elg_keras.h5")
fn = "./240281904_1.jpg"
input_img = cv2.imread(fn)[..., ::-1]
face, lms = fd.detect_face(
input_img) # assuming there is only one face in input image
assert len(face) >= 1, "No face detected"
print(right_eye_xy)
if len(face) > 1:
left_eye_xy = np.array([lms[6][0], lms[1]][0])
right_eye_xy = np.array([lms[5][0], lms[0][0]])
else:
left_eye_xy = np.array([lms[6], lms[1]])
def pressedrunleft(self, instance):
dist_both = self.email.text
### Refer Github Repo GazeML-keras @ https://github.com/shaoanlu/GazeML-keras
### And Github Repo GazeML @ https://github.com/shaoanlu/GazeML-keras
"""## Instantiate GazeML ELG model
#A stacked hourglass framework for iris and eye-lid detection.
"""
model = KerasELG()
model.net.load_weights("./elg_weights/elg_keras.h5")
cnt_one_lf=0
while cnt_one_lf<3:
if cnt_one_lf==0:
fnr = "./one/left/left1.jpg"
right_eye_im = cv2.imread(fnr)[..., ::-1]
dim_x1=right_eye_im.shape[1]
dim_y1=right_eye_im.shape[0]
fnl = "./one/left/left2.jpg"
left_eye_im = cv2.imread(fnl)[..., ::-1]
dim_x2=left_eye_im.shape[1]
dim_y2=left_eye_im.shape[0]
if cnt_one_lf==1:
fnr = "./one/left/left2.jpg"
right_eye_im = cv2.imread(fnr)[..., ::-1]
dim_x1=right_eye_im.shape[1]
dim_y1=right_eye_im.shape[0]
fnl = "./one/left/leftmid.jpg"
left_eye_im = cv2.imread(fnl)[..., ::-1]
dim_x2=left_eye_im.shape[1]
dim_y2=left_eye_im.shape[0]
if cnt_one_lf==2:
fnr = "./one/left/left1.jpg"
right_eye_im = cv2.imread(fnr)[..., ::-1]
dim_x1=right_eye_im.shape[1]
dim_y1=right_eye_im.shape[0]
fnl = "./one/left/leftmid.jpg"
left_eye_im = cv2.imread(fnl)[..., ::-1]
dim_x2=left_eye_im.shape[1]
dim_y2=left_eye_im.shape[0]
pupil_center_right_cp=right_eye_im.copy()
pupil_center_left_cp=left_eye_im.copy()
plt.subplot(1,2,1)
plt.title('Input image')
plt.imshow(right_eye_im)
plt.subplot(1,2,2)
plt.title('Input image')
plt.imshow(left_eye_im)
plt.show()
inp_left = cv2.cvtColor(left_eye_im, cv2.COLOR_RGB2GRAY)
inp_left = cv2.equalizeHist(inp_left)
inp_left = cv2.resize(inp_left, (180,108))[np.newaxis, ..., np.newaxis]
inp_right = cv2.cvtColor(right_eye_im, cv2.COLOR_RGB2GRAY)
inp_right = cv2.equalizeHist(inp_right)
inp_right = cv2.resize(inp_right, (180,108))[np.newaxis, ..., np.newaxis]
## plt.figure(figsize=(8,3))
## plt.subplot(1,2,1)
## #plt.title('Left eye')
## plt.imshow(inp_left[0,...,0], cmap="gray")
## plt.subplot(1,2,2)
## #plt.title('Right eye')
## plt.imshow(inp_right[0,...,0], cmap="gray")
## plt.show()
"""## Predict eye region landmarks
ELG forwardpass. Output shape: (36, 60, 18)
"""
inp_left.shape, inp_right.shape
input_array = np.concatenate([inp_left, inp_right], axis=0)
pred_left, pred_right = model.net.predict(input_array/255 * 2 - 1)
"""## Visualize output heatmaps
Eighteen heatmaps are predicted:
- Eight heatmaps for iris (green)
- Eight heatmaps for eye-lid (red)
- Two heatmaps for pupil (blue)
"""
#@title
plt.figure(figsize=(10,4))
plt.subplot(1,2,1)
plt.axis('off')
plt.title('Left eye heatmaps')
hm_r = np.max(pred_left[...,:8], axis=-1, keepdims=True)
hm_g = np.max(pred_left[...,8:16], axis=-1, keepdims=True)
hm_b = np.max(pred_left[...,16:], axis=-1, keepdims=True)
plt.imshow(np.concatenate([hm_r, hm_g, hm_b], axis=-1))
plt.subplot(1,2,2)
plt.axis('off')
plt.title('Right eye heatmaps')
hm_r = np.max(pred_right[...,:8], axis=-1, keepdims=True)
hm_g = np.max(pred_right[...,8:16], axis=-1, keepdims=True)
hm_b = np.max(pred_right[...,16:], axis=-1, keepdims=True)
plt.imshow(np.concatenate([hm_r, hm_g, hm_b], axis=-1))
plt.show()
"""# Draw eye region landmarks"""
def draw_pupil(im, inp_im, lms):
draw = im.copy()
draw = cv2.resize(draw, (inp_im.shape[2], inp_im.shape[1]))
pupil_center = np.zeros((2,))
pnts_outerline = []
pnts_innerline = []
stroke = inp_im.shape[1] // 12 + 1
for i, lm in enumerate(np.squeeze(lms)):
y, x = int(lm[0]*3), int(lm[1]*3)
if i < 8:
pnts_outerline.append([y, x])
elif i < 16:
pnts_innerline.append([y, x])
pupil_center += (y,x)
pupil_center = (pupil_center/8).astype(np.int32)
draw = cv2.cv2.circle(draw, (pupil_center[0], pupil_center[1]), 1, (255,255,0), -1)
draw = cv2.polylines(draw, [np.array(pnts_outerline).reshape(-1,1,2)], isClosed=True, color=(125,255,125), thickness=stroke//4)
draw = cv2.polylines(draw, [np.array(pnts_innerline).reshape(-1,1,2)], isClosed=True, color=(125,125,255), thickness=stroke//4)
return draw, pupil_center
plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
lms_left = model._calculate_landmarks(pred_left)
result_left, pupil_center_left = draw_pupil(left_eye_im, inp_left, lms_left)
plt.imshow(result_left)
print(pupil_center_left)
plt.subplot(1,2,2)
lms_right = model._calculate_landmarks(pred_right)
result_right, pupil_center_right = draw_pupil(right_eye_im, inp_right, lms_right)
plt.imshow(result_right)
print(pupil_center_right)
plt.show()
plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
ratio_x1=dim_x2/180
print(ratio_x1)
print(pupil_center_left[0])
pupil_center_left[0]=int((pupil_center_left[0])*(ratio_x1))
ratio_y1=dim_y2/108
pupil_center_left[1]=int(pupil_center_left[1]*ratio_y1)
pupil_center_left_cp = cv2.cv2.circle(pupil_center_left_cp, (pupil_center_left[0], pupil_center_left[1]), 10, (255,255,0), -1)
print(pupil_center_left)
plt.imshow(pupil_center_left_cp)
plt.subplot(1,2,2)
ratio_x2=dim_x1/180
ratio_y2=dim_y1/108
pupil_center_right[0]=int(pupil_center_right[0]*ratio_x2)
pupil_center_right[1]=int(pupil_center_right[1]*ratio_y2)
pupil_center_right_cp = cv2.cv2.circle(pupil_center_right_cp, (pupil_center_right[0], pupil_center_right[1]), 10, (255,255,0), -1)
print(pupil_center_right)
plt.imshow(pupil_center_right_cp)
dist_bet_centre = np.linalg.norm(pupil_center_left - pupil_center_right)
print(dist_bet_centre)
print("pixel")
measured=int(self.lastName.text)
max_x_res=int(self.xres.text)
print(round(dist_bet_centre*(measured/max_x_res),2))
pnt_center=[]
#plt.show()
cnt_one_lf+=1
def pressedrune(self, instance):
dist_one = self.lastName.text
### Refer Github Repo GazeML-keras @ https://github.com/shaoanlu/GazeML-keras
### And Github Repo GazeML @ https://github.com/shaoanlu/GazeML-keras
"""## Instantiate GazeML ELG model
##A stacked hourglass framework for iris and eye-lid detection.
"""
model = KerasELG()
model.net.load_weights("./elg_weights/elg_keras.h5")
cnt_both=0
'''Loop over all combination of 3 images'''
while(cnt_both<3):
if cnt_both == 0:
fn1 = "./both/bmd.jpg"
input_img = cv2.imread(fn1)[..., ::-1]
if cnt_both == 1:
fn2 = "./both/brt.jpg"
input_img = cv2.imread(fn2)[..., ::-1]
if cnt_both == 2:
fn3 = "./both/blf.jpg"
input_img = cv2.imread(fn3)[..., ::-1]
self.pb.value=20
im_dim=input_img.shape
## print("dimesnsions of image are:")
## print("X=")
## print(im_dim[1])
## print("Y=")
## print(im_dim[0])
plt.title('Input image')
plt.imshow(input_img)
plt.show()
#########################################################
#cutting in half
left_eye_im = input_img[
int(0):int(im_dim[0]),
int(im_dim[1]/2):int(im_dim[1]), :]
#left_eye_im = left_eye_im[:,::-1,:] # No need for flipping left eye for iris detection
right_eye_im = input_img[
int(0):int(im_dim[0]),
int(0):int(im_dim[1]/2), :]
self.pb.value=30
##########################################################
## plt.figure(figsize=(15,4))
## plt.subplot(1,2,1)
## plt.title('Left eye')
## plt.imshow(left_eye_im)
## plt.subplot(1,2,2)
## plt.title('Right eye')
## plt.imshow(right_eye_im)
## plt.show()
dim=left_eye_im.shape
## print("dimensions are")
## print(dim[0])
## print(dim[1])
"""## Preprocess eye images
ELG has fixed input shape of (108, 180, 1).
Input images are first converted to grey scale, thrown into the histogram equalization process, and finally rescaled to [-1, +1].
"""
inp_left = cv2.cvtColor(left_eye_im, cv2.COLOR_RGB2GRAY)
inp_left = cv2.equalizeHist(inp_left)
inp_left = cv2.resize(inp_left, (180,108))[np.newaxis, ..., np.newaxis]
inp_right = cv2.cvtColor(right_eye_im, cv2.COLOR_RGB2GRAY)
inp_right = cv2.equalizeHist(inp_right)
inp_right = cv2.resize(inp_right, (180,108))[np.newaxis, ..., np.newaxis]
self.pb.value=50
## plt.figure(figsize=(8,3))
## plt.subplot(1,2,1)
## plt.title('Left eye')
## plt.imshow(inp_left[0,...,0], cmap="gray")
## plt.subplot(1,2,2)
## plt.title('Right eye')
## plt.imshow(inp_right[0,...,0], cmap="gray")
## plt.show()
"""## Predict eye region landmarks
ELG forwardpass. Output shape: (36, 60, 18)
"""
inp_left.shape, inp_right.shape
input_array = np.concatenate([inp_left, inp_right], axis=0)
pred_left, pred_right = model.net.predict(input_array/255 * 2 - 1)
"""## Visualize output heatmaps
Eighteen heatmaps are predicted:
- Eight heatmaps for iris (green)
- Eight heatmaps for eye-lid (red)
- Two heatmaps for pupil (blue)
"""
plt.figure(figsize=(10,4))
plt.subplot(1,2,1)
plt.axis('off')
plt.title('Left eye heatmaps')
hm_r = np.max(pred_left[...,:8], axis=-1, keepdims=True)
hm_g = np.max(pred_left[...,8:16], axis=-1, keepdims=True)
hm_b = np.max(pred_left[...,16:], axis=-1, keepdims=True)
plt.imshow(np.concatenate([hm_r, hm_g, hm_b], axis=-1))
plt.subplot(1,2,2)
plt.axis('off')
plt.title('Right eye heatmaps')
hm_r = np.max(pred_right[...,:8], axis=-1, keepdims=True)
hm_g = np.max(pred_right[...,8:16], axis=-1, keepdims=True)
hm_b = np.max(pred_right[...,16:], axis=-1, keepdims=True)
plt.imshow(np.concatenate([hm_r, hm_g, hm_b], axis=-1))
plt.show()
"""# Draw eye region landmarks"""
pnt_center=[]
def draw_pupil(im, inp_im, lms):
draw = im.copy()
draw = cv2.resize(draw, (inp_im.shape[2], inp_im.shape[1]))
pupil_center = np.zeros((2,))
pnts_outerline = []
pnts_innerline = []
stroke = inp_im.shape[1] // 12 + 1
for i, lm in enumerate(np.squeeze(lms)):
#print(lm)
y, x = int(lm[0]*3), int(lm[1]*3)
if i < 8:
#draw = cv2.circle(draw, (y, x), stroke, (125,255,125), -1)
pnts_outerline.append([y, x])
elif i < 16:
#draw = cv2.circle(draw, (y, x), stroke, (125,125,255), -1)
pnts_innerline.append([y, x])
pupil_center += (y,x)
pupil_center = (pupil_center/8).astype(np.int32)
draw = cv2.cv2.circle(draw, (pupil_center[0], pupil_center[1]), stroke//6, (255,255,0), -1)
draw = cv2.polylines(draw, [np.array(pnts_outerline).reshape(-1,1,2)], isClosed=True, color=(125,255,125), thickness=stroke//5)
draw = cv2.polylines(draw, [np.array(pnts_innerline).reshape(-1,1,2)], isClosed=True, color=(125,125,255), thickness=stroke//5)
pnt_center.append(pupil_center[0])
print(pupil_center[0])
pnt_center.append(pupil_center[1])
print(pupil_center[1])
return draw
## plt.figure(figsize=(15,4))
## plt.subplot(1,2,1)
## plt.title("Left eye")
lms_left = model._calculate_landmarks(pred_left)
result_left = draw_pupil(left_eye_im, inp_left, lms_left)
## plt.imshow(result_left)
## plt.subplot(1,2,2)
## plt.title("Right eye")
lms_right = model._calculate_landmarks(pred_right)
result_right = draw_pupil(right_eye_im, inp_right, lms_right)
## plt.imshow(result_right)
## plt.show()
draw2 = input_img.copy()
dim2=input_img.shape
'''Mapping Back to Original Img'''
a=int(pnt_center[1]*(dim[0]/108))
b=int(pnt_center[0]*(dim[1]/180) + int(im_dim[1]/2))
c=int(pnt_center[3]*(dim[0]/108))
d=int(pnt_center[2]*(dim[1]/180))
print("left_eye Y")
print(a)
print("left_eye X")
print(b)
print("right_eye Y")
print(c)
print("right_eye X")
print(d)
slice_h = slice(int(0), int(im_dim[0]))
slice_w = slice(int(im_dim[1]/2), int(im_dim[1]))
im_shape = left_eye_im.shape[::-1]
draw2[slice_h, slice_w, :] = cv2.resize(result_left, im_shape[1:])
slice_h = slice(int(0), int(im_dim[0]))
slice_w = slice(int(0), int(im_dim[1]/2))
im_shape = right_eye_im.shape[::-1]
net_dist=math.sqrt((a-c)*(a-c)+(b-d)*(b-d))
draw2[slice_h, slice_w, :] = cv2.resize(result_right, im_shape[1:])
plt.imshow(draw2)
## plt.show()
print("The net Distance between the pupil Centres is :")
print(net_dist)
measured=int(self.lastName.text)
max_x_res=int(self.xres.text)
print(round(net_dist*(measured/max_x_res),2))
draw3=cv2.line(draw2, (b, a), (d, c), (255, 0, 0), 2)
dim3=draw2.shape
pnt_center=[]
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(draw3,str(net_dist),(0,0), font, 20,(0,255,0),2,cv2.LINE_AA)
plt.figure(figsize=(10,10))
## plt.imshow(draw2)
## plt.show()
plt.imshow(draw3)
plt.show()
cv2.waitKey(1)
cnt_both+=1
def pressedrunright(self, instance):
name = self.name.text
dist_both = self.email.text
print("Name:", name)
"""## Instantiate GazeML ELG model
#A stacked hourglass framework for iris and eye-lid detection.
"""
model = KerasELG()
model.net.load_weights("./elg_weights/elg_keras.h5")
cnt_one=0
while cnt_one<3:
if cnt_one==0:
fnr = "./one/right/right1.jpg"
right_eye_im = cv2.imread(fnr)[..., ::-1]
dim_x1=right_eye_im.shape[1]
dim_y1=right_eye_im.shape[0]
fnl = "./one/right/right2.jpg"
left_eye_im = cv2.imread(fnl)[..., ::-1]
dim_x2=left_eye_im.shape[1]
dim_y2=left_eye_im.shape[0]
img_name_lf = "./output/right/left.jpg"
img_name_rt = "./output/right/rt.jpg"
makeanim(right_eye_im,left_eye_im,'./anim/one/right_lf_rt')
if cnt_one==1:
fnr = "./one/right/right2.jpg"
right_eye_im = cv2.imread(fnr)[..., ::-1]
dim_x1=right_eye_im.shape[1]
dim_y1=right_eye_im.shape[0]
fnl = "./one/right/rightmid.jpg"
left_eye_im = cv2.imread(fnl)[..., ::-1]
dim_x2=left_eye_im.shape[1]
dim_y2=left_eye_im.shape[0]
img_name_md = "./output/right/mid.jpg"
makeanim(right_eye_im,left_eye_im,'./anim/one/right_lf_md')
img_name = "./output/right/rightmid2.jpg"
if cnt_one==2:
fnr = "./one/right/right1.jpg"
right_eye_im = cv2.imread(fnr)[..., ::-1]
dim_x1=right_eye_im.shape[1]
dim_y1=right_eye_im.shape[0]
fnl = "./one/right/rightmid.jpg"
left_eye_im = cv2.imread(fnl)[..., ::-1]
dim_x2=left_eye_im.shape[1]
dim_y2=left_eye_im.shape[0]
img_name = "./output/right/rightmid1.jpg"
makeanim(right_eye_im,left_eye_im,'./anim/one/right_rt_md')
pupil_center_right_cp=right_eye_im.copy()
pupil_center_left_cp=left_eye_im.copy()
#########################################################
width = pupil_center_right_cp.shape[1]
height = pupil_center_right_cp.shape[0]
FPS = 1
seconds = 2
fourcc = VideoWriter_fourcc(*'MP42')
video = VideoWriter('./circle_noise.avi', fourcc, float(FPS), (width, height))
a=0
for _ in range(FPS*seconds):
if a%2==0:
video.write(pupil_center_left_cp)
else:
video.write(pupil_center_right_cp)
a+=1
video.release()
#########################################################
plt.subplot(1,2,1)
plt.title('Input image rt')
plt.imshow(right_eye_im)
plt.subplot(1,2,2)
plt.title('Input image lf')
plt.imshow(left_eye_im)
plt.show()
inp_left = cv2.cvtColor(left_eye_im, cv2.COLOR_RGB2GRAY)
inp_left = cv2.equalizeHist(inp_left)
inp_left = cv2.resize(inp_left, (180,108))[np.newaxis, ..., np.newaxis]
inp_right = cv2.cvtColor(right_eye_im, cv2.COLOR_RGB2GRAY)
inp_right = cv2.equalizeHist(inp_right)
inp_right = cv2.resize(inp_right, (180,108))[np.newaxis, ..., np.newaxis]
plt.figure(figsize=(8,3))
plt.subplot(1,2,1)
plt.title('Left eye')
plt.imshow(inp_left[0,...,0], cmap="gray")
plt.subplot(1,2,2)
plt.title('Right eye')
plt.imshow(inp_right[0,...,0], cmap="gray")
plt.show()
"""## Predict eye region landmarks
ELG forwardpass. Output shape: (36, 60, 18)
"""
inp_left.shape, inp_right.shape
input_array = np.concatenate([inp_left, inp_right], axis=0)
pred_left, pred_right = model.net.predict(input_array/255 * 2 - 1)
"""## Visualize output heatmaps
Eighteen heatmaps are predicted:
- Eight heatmaps for iris (green)
- Eight heatmaps for eye-lid (red)
- Two heatmaps for pupil (blue)
"""
#@title
plt.figure(figsize=(10,4))
plt.subplot(1,2,1)
plt.axis('off')
plt.title('Left eye heatmaps')
hm_r = np.max(pred_left[...,:8], axis=-1, keepdims=True)
hm_g = np.max(pred_left[...,8:16], axis=-1, keepdims=True)
hm_b = np.max(pred_left[...,16:], axis=-1, keepdims=True)
plt.imshow(np.concatenate([hm_r, hm_g, hm_b], axis=-1))
plt.subplot(1,2,2)
plt.axis('off')
plt.title('Right eye heatmaps')
hm_r = np.max(pred_right[...,:8], axis=-1, keepdims=True)
hm_g = np.max(pred_right[...,8:16], axis=-1, keepdims=True)
hm_b = np.max(pred_right[...,16:], axis=-1, keepdims=True)
plt.imshow(np.concatenate([hm_r, hm_g, hm_b], axis=-1))
plt.show()
"""# Draw eye region landmarks"""
def draw_pupil(im, inp_im, lms):
draw = im.copy()
draw = cv2.resize(draw, (inp_im.shape[2], inp_im.shape[1]))
pupil_center = np.zeros((2,))
pnts_outerline = []
pnts_innerline = []
stroke = inp_im.shape[1] // 12 + 1
for i, lm in enumerate(np.squeeze(lms)):
y, x = int(lm[0]*3), int(lm[1]*3)
if i < 8:
pnts_outerline.append([y, x])
elif i < 16:
pnts_innerline.append([y, x])
pupil_center += (y,x)
pupil_center = (pupil_center/8).astype(np.int32)
draw = cv2.cv2.circle(draw, (pupil_center[0], pupil_center[1]), 1, (255,255,0), -1)
draw = cv2.polylines(draw, [np.array(pnts_outerline).reshape(-1,1,2)], isClosed=True, color=(125,255,125), thickness=stroke//4)
draw = cv2.polylines(draw, [np.array(pnts_innerline).reshape(-1,1,2)], isClosed=True, color=(125,125,255), thickness=stroke//4)
return draw, pupil_center
plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
plt.title("Left eye")
lms_left = model._calculate_landmarks(pred_left)
result_left, pupil_center_left = draw_pupil(left_eye_im, inp_left, lms_left)
plt.imshow(result_left)
#print(pupil_center_left)
plt.subplot(1,2,2)
plt.title("Right eye")
lms_right = model._calculate_landmarks(pred_right)
result_right, pupil_center_right = draw_pupil(right_eye_im, inp_right, lms_right)
plt.imshow(result_right)
#print(pupil_center_right)
plt.show()
plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
ratio_x1=dim_x2/180
#print(ratio_x1)
#print(pupil_center_left[0])
pupil_center_left[0]=int((pupil_center_left[0])*(ratio_x1))
ratio_y1=dim_y2/108
pupil_center_left[1]=int(pupil_center_left[1]*ratio_y1)
pupil_center_left_cp = cv2.cv2.circle(pupil_center_left_cp, (pupil_center_left[0], pupil_center_left[1]), 10, (255,255,0), -1)
print(pupil_center_left)
plt.imshow(pupil_center_left_cp)
plt.subplot(1,2,2)
ratio_x2=dim_x1/180
ratio_y2=dim_y1/108
pupil_center_right[0]=int(pupil_center_right[0]*ratio_x2)
pupil_center_right[1]=int(pupil_center_right[1]*ratio_y2)
pupil_center_right_cp = cv2.cv2.circle(pupil_center_right_cp, (pupil_center_right[0], pupil_center_right[1]), 10, (255,255,0), -1)
print(pupil_center_right)
plt.imshow(pupil_center_right_cp)
dist_bet_centre = np.linalg.norm(pupil_center_left - pupil_center_right)
print(dist_bet_centre)
print("pixel")
measured=int(self.lastName.text)
max_x_res=int(self.xres.text)
print(round(dist_bet_centre*(measured/max_x_res),2))
print("in mm")
if cnt_one==0:
cv2.imwrite(img_name_lf, pupil_center_left_cp)
cv2.imwrite(img_name_rt, pupil_center_right_cp)
if cnt_one==1:
cv2.imwrite(img_name_md, pupil_center_right_cp)
pnt_center=[]
#plt.show()
## if cnt_one==0:
## a=str(round(dist_bet_centre*(measured/max_x_res),2))
## elif cnt_one==1:
## b=str(round(dist_bet_centre*(measured/max_x_res),2))
## elif cnt_one==2:
## c=str(round(dist_bet_centre*(measured/max_x_res),2))
## data['dist'].append({
## '1': str(a),
## '2': str(b),
## '3': str(c)
## })
cnt_one+=1