def scaleFn(self, img, label, elParam, pupil_center):
dsize = (int(self.scale * img.shape[1]),
int(self.scale * img.shape[0]))
H = np.array([[self.scale, 0, 0], [0, self.scale, 0], [0, 0, 1]])
img = cv2.resize(img, dsize, interpolation=cv2.INTER_LANCZOS4)
label = cv2.resize(label, dsize, interpolation=cv2.INTER_NEAREST)
elParam_1 = my_ellipse(elParam[0]).transform(H)[0][:-1] if not np.all(
elParam[0] == -1) else elParam[0]
elParam_2 = my_ellipse(elParam[1]).transform(H)[0][:-1] if not np.all(
elParam[0] == -1) else elParam[0]
elParam = (elParam_1, elParam_2)
pupil_center = H[:2, :2].dot(pupil_center) if not np.all(
pupil_center == -1) else pupil_center
return img, label, elParam, pupil_center
def get_iris_pupil_center_from_eye_parts(data_to_fit):
pupilPts, irisPts = getValidPoints(data_to_fit)
if len(pupilPts) > 0:
model_pupil = ransac(pupilPts, ElliFit, 15, 40, 5e-3, 15).loop()
pupil_fit_error = my_ellipse(model_pupil.model).verify(pupilPts)
r, c = np.where(data_to_fit == 2)
pupil_loc = model_pupil.model[:
2] if pupil_fit_error < 0.05 else np.stack(
[np.mean(c), np.mean(r)], axis=0)
else:
pupil_loc = None
# Iris ellipse fit
if len(irisPts) > 0:
model_iris = ransac(irisPts, ElliFit, 15, 40, 5e-3, 15).loop()
iris_fit_error = my_ellipse(model_iris.model).verify(irisPts)
iris_loc = model_iris.model[0:2]
else:
iris_loc = None
return pupil_loc, iris_loc
c = int(0.5 * (np.max(r) + np.min(r)))
top, bot = (0, c + 150 - (c - 150)) if c - 150 < 0 else (c - 150,
c + 150)
I = I[top:bot, :]
LabelMat = LabelMat[top:bot, :]
I = cv2.resize(I, (640, 480), interpolation=cv2.INTER_LANCZOS4)
LabelMat = cv2.resize(LabelMat, (640, 480),
interpolation=cv2.INTER_NEAREST)
#%%
pupilPts, irisPts = getValidPoints(LabelMat)
if np.sum(LabelMat == 3) > 150 and type(pupilPts) is not list:
model_pupil = ransac(pupilPts, ElliFit, 15, 40, 5e-3,
15).loop()
pupil_fit_error = my_ellipse(
model_pupil.model).verify(pupilPts)
else:
print('Not enough pupil points')
model_pupil = type('model', (object, ), {})
model_pupil.model = np.array([-1, -1, -1, -1, -1])
pupil_fit_error = np.inf
if np.sum(LabelMat == 2) > 200 and type(irisPts) is not list:
model_iris = ransac(irisPts, ElliFit, 15, 40, 5e-3, 15).loop()
iris_fit_error = my_ellipse(model_iris.model).verify(irisPts)
else:
print('Not enough iris points')
model_iris = type('model', (object, ), {})
model_iris.model = np.array([-1, -1, -1, -1, -1])
model_iris.Phi = np.array([-1, -1, -1, -1, -1])
iris_fit_error = np.inf
def evaluate_ellseg_on_image(frame, model):
assert len(frame.shape) == 4, 'Frame must be [1,1,H,W]'
with torch.no_grad():
x4, x3, x2, x1, x = model.enc(frame)
latent = torch.mean(x.flatten(start_dim=2), -1)
elOut = model.elReg(x, 0)
seg_out = model.dec(x4, x3, x2, x1, x)
seg_out, elOut, latent = seg_out.cpu(), elOut.squeeze().cpu(
), latent.squeeze().cpu()
seg_map = get_predictions(seg_out).squeeze().numpy()
ellipse_from_network = 1 if args.ellseg_ellipses == 1 else 0
ellipse_from_output = 1 if args.ellseg_ellipses == 0 else 0
no_ellipse = 1 if args.ellseg_ellipses == -1 else 0
if ellipse_from_network:
# Get EllSeg proposed ellipse predictions
# Ellipse Centers -> derived from segmentation output
# Ellipse axes and orientation -> Derived from latent space
_, norm_pupil_center = get_seg2ptLoss(seg_out[:, 2, ...],
torch.zeros(2, ),
temperature=4)
_, norm_iris_center = get_seg2ptLoss(-seg_out[:, 0, ...],
torch.zeros(2, ),
temperature=4)
norm_pupil_ellipse = torch.cat([norm_pupil_center, elOut[7:10]])
norm_iris_ellipse = torch.cat([norm_iris_center, elOut[2:5]])
# Transformation function H
_, _, H, W = frame.shape
H = np.array([[W / 2, 0, W / 2], [0, H / 2, H / 2], [0, 0, 1]])
pupil_ellipse = my_ellipse(
norm_pupil_ellipse.numpy()).transform(H)[0][:-1]
iris_ellipse = my_ellipse(
norm_iris_ellipse.numpy()).transform(H)[0][:-1]
if ellipse_from_output:
# Get ElliFit derived ellipse fits from segmentation mask
seg_map_temp = copy.deepcopy(seg_map)
seg_map_temp[seg_map_temp == 2] += 1 # Pupil by PartSeg standard is 3
seg_map_temp[seg_map_temp == 1] += 1 # Iris by PartSeg standard is 2
pupilPts, irisPts = getValidPoints(seg_map_temp, isPartSeg=False)
if np.sum(seg_map_temp == 3) > 50 and type(pupilPts) is not list:
if args.skip_ransac:
model_pupil = ElliFit(**{'data': pupilPts})
else:
model_pupil = ransac(pupilPts, ElliFit, 15, 40, 5e-3,
15).loop()
else:
print('Not enough pupil points')
model_pupil = type('model', (object, ), {})
model_pupil.model = np.array([-1, -1, -1, -1, -1])
if np.sum(seg_map_temp == 2) > 50 and type(irisPts) is not list:
if args.skip_ransac:
model_iris = ElliFit(**{'data': irisPts})
else:
model_iris = ransac(irisPts, ElliFit, 15, 40, 5e-3, 15).loop()
else:
print('Not enough iris points')
model_iris = type('model', (object, ), {})
model_iris.model = np.array([-1, -1, -1, -1, -1])
model_iris.Phi = np.array([-1, -1, -1, -1, -1])
# iris_fit_error = np.inf
pupil_ellipse = np.array(model_pupil.model)
iris_ellipse = np.array(model_iris.model)
if no_ellipse:
pupil_ellipse = np.array([-1, -1, -1, -1, -1])
iris_ellipse = np.array([-1, -1, -1, -1, -1])
return seg_map, latent.cpu().numpy(), pupil_ellipse, iris_ellipse
def get_mask_from_cv2_image(image,
model,
useGpu=True,
pupilOnly=False,
includeRawPredict=False,
channels=3,
trim_pupil=False,
isEllseg=False,
ellsegPrecision=None,
useEllsegEllipseAsMask=False):
if useGpu:
device = torch.device("cuda")
else:
device = torch.device("cpu")
if not isEllseg:
img = image.unsqueeze(1)
data = img.to(device)
output = model(data)
rawpredict = get_predictions(output)
predict = rawpredict + 1
# print(np.unique(predict[0].cpu().numpy()))
pred_img = 1 - predict[0].cpu().numpy() / channels
else:
img = np.array(Image.fromarray(image).convert("L"))
img = (img - img.mean()) / img.std()
img = torch.from_numpy(img).unsqueeze(0).to(
ellsegPrecision) # Adds a singleton for channels
img = img.unsqueeze(0)
img = img.to(device).to(ellsegPrecision)
x4, x3, x2, x1, x = model.enc(img)
op = model.dec(x4, x3, x2, x1, x)
rawpredict = get_predictions(op)
plt.imshow(rawpredict[0], cmap="BrBG", alpha=0.3)
if useEllsegEllipseAsMask:
ellpred = model.elReg(x, 0).view(-1)
#i1, i2, i3, i4, i5, p1, p2, p3, p4, p5 = ellpred[0].cpu().detach().numpy()
_, _, H, W = img.shape
H_mat = np.array([[W / 2, 0, W / 2], [0, H / 2, H / 2], [0, 0, 1]])
#import pdb
#pdb.set_trace()
i_cx, i_cy, i_a, i_b, i_theta, _ = my_ellipse(
ellpred[:5].tolist()).transform(H_mat)[0]
p_cx, p_cy, p_a, p_b, p_theta, _ = my_ellipse(
ellpred[5:].tolist()).transform(H_mat)[0]
ellimage = np.full((int(H), int(W)), 2 / 3)
startAngle = 0
endAngle = 360
iris_color = 1 / 3
pupil_color = 0.0
pred_img = draw_ellipse(ellimage, (i_cx, i_cy), (i_a, i_b),
i_theta, startAngle, endAngle, iris_color,
-1)
pred_img = draw_ellipse(ellimage, (p_cx, p_cy), (p_a, p_b),
p_theta, startAngle, endAngle, pupil_color,
-1)
else:
predict = rawpredict + 1
pred_img = 1 - predict[0].cpu().numpy() / channels
#print(pred_img)
# trim pupil if asked to
if trim_pupil:
newimg = np.invert(pred_img > 0)
labeled_img = measure.label(newimg)
labels = np.unique(labeled_img)
newimg = np.zeros((newimg.shape[0], newimg.shape[1]))
old_sum = 0
old_label = None
for label in (y for y in labels if y != 0):
if np.sum(labeled_img == label) > old_sum:
old_sum = np.sum(labeled_img == label)
old_label = label
if old_label is not None:
newimg = newimg + (labeled_img == old_label)
newimg[newimg == 0] = 2
newimg[newimg == 1] = 0
newimg[newimg == 2] = 1
pred_img[pred_img == 0] = 1 - (1 / channels)
pred_img[newimg == 0] = 0
#print(np.unique(pred_img))
if pupilOnly:
pred_img = np.ceil(pred_img) * 0.5
if includeRawPredict:
return pred_img, rawpredict
return pred_img
def get_pupil_ellipse_from_cv2_image(image,
model,
useGpu=True,
predict=None,
isEllseg=False,
ellsegPrecision=None,
ellsegEllipse=False,
debugWindowName=None):
"""
OUTPUT FORMAT
{
0: center x,
1: center y,
2: ellipse major axis radius,
3: ellipse minor axis radius,
4: ellipse angle
}
"""
if useGpu:
device = torch.device("cuda")
else:
device = torch.device("cpu")
if predict is None:
if not isEllseg:
img = image.unsqueeze(1)
data = img.to(device)
output = model(data)
predict = get_predictions(output)
pred_img = predict[0].numpy()
else: # w:320 h:240
img = np.array(transforms.ToPILImage()(image).convert("L"))
img = (img - img.mean()) / img.std()
img = torch.from_numpy(img).unsqueeze(0).to(
ellsegPrecision) # Adds a singleton for channels
img = img.unsqueeze(0)
img = img.to(device).to(ellsegPrecision)
x4, x3, x2, x1, x = model.enc(img)
op = model.dec(x4, x3, x2, x1, x)
if ellsegEllipse: # Option to get ellipse directly from ellseg output
ellpred = model.elReg(x, 0).view(-1)
_, _, H, W = img.shape
H_mat = np.array([[W / 2, 0, W / 2], [0, H / 2, H / 2],
[0, 0, 1]])
p_cx, p_cy, p_a, p_b, p_theta, _ = my_ellipse(
ellpred[5:].tolist()).transform(H_mat)[0]
return [p_cx, p_cy, p_a, p_b, p_theta]
# [centerX, centerY, axis1, axis2, angle]
#elOut = model.elReg(x, 0) # Linear regression to ellipse parameters
#print(elOut.shape)
predict = get_predictions(op)
pred_img = predict[0].numpy()
# cv2.imshow("ELLIPSE", pred_img/2)
else:
pred_img = predict[0].numpy()
if debugWindowName is not None:
outIm = pred_img / np.max(pred_img)
cv2.imshow(debugWindowName, outIm)
return get_pupil_parameters(pred_img)
def generateImageGrid(I,
mask,
elNorm,
pupil_center,
cond,
heatmaps=False,
override=False):
'''
Parameters
----------
I : numpy array [B, H, W]
A batchfirst array which holds images
mask : numpy array [B, H, W]
A batch first array which holds for individual pixels.
hMaps: numpy array [B, C, N, H, W]
N is the # of points, C is the category the points belong to (iris or
pupil). Heatmaps are gaussians centered around point of interest
elNorm:numpy array [B, C, 5]
Normalized ellipse parameters
pupil_center : numpy array [B, 2]
Identified pupil center for plotting.
cond : numpy array [B, 5]
A flag array which holds information about what information is present.
heatmaps : bool, optional
Unless specificed, does not show the heatmaps of predicted points
override : bool, optional
An override flag which plots data despite being demarked in the flag
array. Generally used during testing.
The default is False.
Returns
-------
I_o : numpy array [Ho, Wo]
Returns an array holding concatenated images from the input overlayed
with segmentation mask, pupil center and pupil ellipse.
Note: If masks exist, then ellipse parameters would exist aswell.
'''
B, H, W = I.shape
mesh = create_meshgrid(H, W, normalized_coordinates=True) # 1xHxWx2
H = np.array([[W / 2, 0, W / 2], [0, H / 2, H / 2], [0, 0, 1]])
I_o = []
for i in range(0, min(16, cond.shape[0])):
im = I[i, ...].squeeze() - I[i, ...].min()
im = cv2.equalizeHist(np.uint8(255 * im / im.max()))
im = np.stack([im for i in range(0, 3)], axis=2)
if (not cond[i, 1]) or override:
# If masks exists
rr, cc = np.where(mask[i, ...] == 1)
im[rr, cc, ...] = np.array([0, 255, 0]) # Green
rr, cc = np.where(mask[i, ...] == 2)
im[rr, cc, ...] = np.array([255, 255, 0]) # Yellow
# Just for experiments. Please ignore.
el_iris = elNorm[i, 0, ...]
X = (mesh[..., 0].squeeze() - el_iris[0])*np.cos(el_iris[-1])+\
(mesh[..., 1].squeeze() - el_iris[1])*np.sin(el_iris[-1])
Y = -(mesh[..., 0].squeeze() - el_iris[0])*np.sin(el_iris[-1])+\
(mesh[..., 1].squeeze() - el_iris[1])*np.cos(el_iris[-1])
wtMat = (X / el_iris[2])**2 + (Y / el_iris[3])**2 - 1
# [rr_i, cc_i] = np.where(wtMat< 0)
try:
el_iris = my_ellipse(elNorm[i, 0, ...]).transform(H)[0]
el_pupil = my_ellipse(elNorm[i, 1, ...]).transform(H)[0]
except:
print(
'Warning: inappropriate ellipses. Defaulting to not break runtime..'
)
el_iris = np.array([W / 2, H / 2, W / 8, H / 8,
0.0]).astype(np.float32)
el_pupil = np.array([W / 2, H / 2, W / 4, H / 4,
0.0]).astype(np.float32)
[rr_i, cc_i] = draw.ellipse_perimeter(int(el_iris[1]),
int(el_iris[0]),
int(el_iris[3]),
int(el_iris[2]),
orientation=el_iris[4])
[rr_p, cc_p] = draw.ellipse_perimeter(int(el_pupil[1]),
int(el_pupil[0]),
int(el_pupil[3]),
int(el_pupil[2]),
orientation=el_pupil[4])
rr_i = rr_i.clip(6, im.shape[0] - 6)
rr_p = rr_p.clip(6, im.shape[0] - 6)
cc_i = cc_i.clip(6, im.shape[1] - 6)
cc_p = cc_p.clip(6, im.shape[1] - 6)
im[rr_i, cc_i, ...] = np.array([0, 0, 255])
im[rr_p, cc_p, ...] = np.array([255, 0, 0])
if (not cond[i, 0]) or override:
# If pupil center exists
rr, cc = draw.disk((pupil_center[i, 1].clamp(6, im.shape[0] - 6),
pupil_center[i, 0].clamp(6, im.shape[1] - 6)),
5)
im[rr, cc, ...] = 255
I_o.append(im)
I_o = np.stack(I_o, axis=0)
I_o = np.moveaxis(I_o, 3, 1)
I_o = make_grid(torch.from_numpy(I_o).to(torch.float), nrow=4)
I_o = I_o - I_o.min()
I_o = I_o / I_o.max()
return I_o
Data['Images'].append(I)
Data['Masks'].append(mask)
Data['Masks_noSkin'].append(mask_noSkin)
Data['Info'].append(str_imName)
keydict['resolution'].append(I.shape)
keydict['archive'].append(ds_name)
temp = 255 * (mask_noSkin == 3)
edge = cv2.Canny(temp.astype(np.uint8), 10, 150) + cv2.Canny(
(255 - temp).astype(np.uint8), 10, 150)
r, c = np.where(edge)
temp_pts = np.stack([c, r], axis=1) # Improve readability
model_pupil = ransac(temp_pts, ElliFit, 15, 10, 0.05,
np.round(temp_pts.shape[0] / 2)).loop()
pupil_fit_error = my_ellipse(model_pupil.model).verify(temp_pts)
pupil_loc = model_pupil.model[:2]
temp = 255 * ((mask_noSkin == 2) | (mask_noSkin == 3))
edge = cv2.Canny(temp.astype(np.uint8), 10, 150) + cv2.Canny(
(255 - temp).astype(np.uint8), 10, 150)
r, c = np.where(edge)
temp_pts = np.stack([c, r], axis=1)
model_iris = ransac(temp_pts, ElliFit, 15, 10, 0.05,
np.round(temp_pts.shape[0] / 2)).loop()
iris_fit_error = my_ellipse(model_iris.model).verify(temp_pts)
Data['Fits']['pupil'].append(model_pupil.model)
Data['Fits']['iris'].append(model_iris.model)
# Ellipse values defined in eucid coordinates (Y axis up)
cx = 180
cy = 180
a = 120
b = 60
deg = 30
[rr, cc] = ellipse(cy, cx, b, a, rotation=np.deg2rad(-deg))
I = np.zeros((400, 400))
I[rr, cc] = 1
H = np.array([[2 / 20, 0, -1], [0, 2 / 20, -1], [0, 0, 1]])
e1 = np.array([cx, cy, a, b, np.deg2rad(deg)])
p1 = my_ellipse(e1)
e2 = p1.transform(H)[0][:-1]
p2 = my_ellipse(e2)
e3 = p2.transform(np.linalg.inv(H))[0][:-1]
print('Original ellipse: {}'.format(np.round(e1, 4)))
print('Normalized ellipse: {}'.format(e2))
print('Recon ellipse: {}'.format(np.round(e3, 4)))
pts1 = p1.generatePoints(50, 'equiAngle')
pts2 = p2.generatePoints(50, 'equiAngle')
N = len(pts1[0])
fit1 = ElliFit(**{'data': np.stack(pts1, 1)})
fit2 = ElliFit(**{'data': np.stack(pts2, 1)})
cone_params = [a,b,c,d,f,g,h,u,v,w]
return M, cone_params, eig_vals
H5_file = 'riteyes_s-natural_15_4.h5'
path_data = '/media/rakshit/Monster/Datasets/All'
f = h5py.File(os.path.join(path_data, H5_file), 'r')
im_num = 143
mask = f['Masks_noSkin'][im_num, ...]
image = f['Images'][im_num, ...]
iris_ellipse = f['Fits']['iris'][im_num, ...]
f.close()
iris_quad = my_ellipse(iris_ellipse.tolist()).quad.tolist()
a, hh, b, gg, ff, d = iris_quad
h, g, f = hh/2, gg/2, ff/2
ellipse_quad = torch.tensor([a, h, b, g, f, d])
#%% Step 1 - Get conic equation
M, cone_params, eig_vals = get_cone_quad(ellipse_quad)
#%% Step 2 - Get reduce conic equation
# alphas = get_reduced(cone_param)
def getPts(param):
ellipse_mod = my_ellipse(param)
x_1, y_1 = ellipse_mod.generatePoints(50, 'equiSlope')
x_2, y_2 = ellipse_mod.generatePoints(50, 'equiAngle')
return np.concatenate([x_1, x_2]), np.concatenate([y_1, y_2])
return patches
n_pts = 5
I_res = (240, 240)
H = np.array([[2/240, 0, -1], [0, 2/240, -1], [0, 0, 1]])
angleItr = np.linspace(10, 170, n_pts)
eccItr = np.linspace(0.4, 1.6, n_pts) + 1e-5 # Never a perfect 1
data = {'I':[], 'xPts':[], 'yPts':[]}
for ang in np.nditer(angleItr):
for ecc in np.nditer(eccItr):
# Input as param
param = np.array([130, 110, 40, 40*ecc, np.deg2rad(ang)])
#print('Phi: {}'.format(np.round(my_ellipse(param).recover_Phi(), 2)))
param_norm = my_ellipse(param).transform(H)[0][:5]
print('Theta: {}. E: {}'.format(ang, ecc))
print('Param: {}'.format(np.round(param, 2)))
print('Param norm: {}'.format(np.round(param_norm, 2)))
print('Phi: {}'.format(np.round(my_ellipse(param_norm).recover_Phi(), 2)))
x_pts, y_pts = getPts(param)
I = np.zeros(I_res)
rr, cc = ellipse(param[1], param[0], param[3], param[2], shape=I_res, rotation=param[4])
I[rr, cc] = 1
data['I'].append(I)
data['xPts'].append(x_pts)
data['yPts'].append(y_pts)
data['I'] = np.stack(data['I'], axis=0)
data['xPts'] = np.stack(data['xPts'], axis=0)
data['yPts'] = np.stack(data['yPts'], axis=0)
