def scaleFn(self, img, label, elParam, pupil_center): dsize = (int(self.scale * img.shape[1]), int(self.scale * img.shape[0])) H = np.array([[self.scale, 0, 0], [0, self.scale, 0], [0, 0, 1]]) img = cv2.resize(img, dsize, interpolation=cv2.INTER_LANCZOS4) label = cv2.resize(label, dsize, interpolation=cv2.INTER_NEAREST) elParam_1 = my_ellipse(elParam[0]).transform(H)[0][:-1] if not np.all( elParam[0] == -1) else elParam[0] elParam_2 = my_ellipse(elParam[1]).transform(H)[0][:-1] if not np.all( elParam[0] == -1) else elParam[0] elParam = (elParam_1, elParam_2) pupil_center = H[:2, :2].dot(pupil_center) if not np.all( pupil_center == -1) else pupil_center return img, label, elParam, pupil_center
def get_iris_pupil_center_from_eye_parts(data_to_fit): pupilPts, irisPts = getValidPoints(data_to_fit) if len(pupilPts) > 0: model_pupil = ransac(pupilPts, ElliFit, 15, 40, 5e-3, 15).loop() pupil_fit_error = my_ellipse(model_pupil.model).verify(pupilPts) r, c = np.where(data_to_fit == 2) pupil_loc = model_pupil.model[: 2] if pupil_fit_error < 0.05 else np.stack( [np.mean(c), np.mean(r)], axis=0) else: pupil_loc = None # Iris ellipse fit if len(irisPts) > 0: model_iris = ransac(irisPts, ElliFit, 15, 40, 5e-3, 15).loop() iris_fit_error = my_ellipse(model_iris.model).verify(irisPts) iris_loc = model_iris.model[0:2] else: iris_loc = None return pupil_loc, iris_loc
c = int(0.5 * (np.max(r) + np.min(r))) top, bot = (0, c + 150 - (c - 150)) if c - 150 < 0 else (c - 150, c + 150) I = I[top:bot, :] LabelMat = LabelMat[top:bot, :] I = cv2.resize(I, (640, 480), interpolation=cv2.INTER_LANCZOS4) LabelMat = cv2.resize(LabelMat, (640, 480), interpolation=cv2.INTER_NEAREST) #%% pupilPts, irisPts = getValidPoints(LabelMat) if np.sum(LabelMat == 3) > 150 and type(pupilPts) is not list: model_pupil = ransac(pupilPts, ElliFit, 15, 40, 5e-3, 15).loop() pupil_fit_error = my_ellipse( model_pupil.model).verify(pupilPts) else: print('Not enough pupil points') model_pupil = type('model', (object, ), {}) model_pupil.model = np.array([-1, -1, -1, -1, -1]) pupil_fit_error = np.inf if np.sum(LabelMat == 2) > 200 and type(irisPts) is not list: model_iris = ransac(irisPts, ElliFit, 15, 40, 5e-3, 15).loop() iris_fit_error = my_ellipse(model_iris.model).verify(irisPts) else: print('Not enough iris points') model_iris = type('model', (object, ), {}) model_iris.model = np.array([-1, -1, -1, -1, -1]) model_iris.Phi = np.array([-1, -1, -1, -1, -1]) iris_fit_error = np.inf
def evaluate_ellseg_on_image(frame, model): assert len(frame.shape) == 4, 'Frame must be [1,1,H,W]' with torch.no_grad(): x4, x3, x2, x1, x = model.enc(frame) latent = torch.mean(x.flatten(start_dim=2), -1) elOut = model.elReg(x, 0) seg_out = model.dec(x4, x3, x2, x1, x) seg_out, elOut, latent = seg_out.cpu(), elOut.squeeze().cpu( ), latent.squeeze().cpu() seg_map = get_predictions(seg_out).squeeze().numpy() ellipse_from_network = 1 if args.ellseg_ellipses == 1 else 0 ellipse_from_output = 1 if args.ellseg_ellipses == 0 else 0 no_ellipse = 1 if args.ellseg_ellipses == -1 else 0 if ellipse_from_network: # Get EllSeg proposed ellipse predictions # Ellipse Centers -> derived from segmentation output # Ellipse axes and orientation -> Derived from latent space _, norm_pupil_center = get_seg2ptLoss(seg_out[:, 2, ...], torch.zeros(2, ), temperature=4) _, norm_iris_center = get_seg2ptLoss(-seg_out[:, 0, ...], torch.zeros(2, ), temperature=4) norm_pupil_ellipse = torch.cat([norm_pupil_center, elOut[7:10]]) norm_iris_ellipse = torch.cat([norm_iris_center, elOut[2:5]]) # Transformation function H _, _, H, W = frame.shape H = np.array([[W / 2, 0, W / 2], [0, H / 2, H / 2], [0, 0, 1]]) pupil_ellipse = my_ellipse( norm_pupil_ellipse.numpy()).transform(H)[0][:-1] iris_ellipse = my_ellipse( norm_iris_ellipse.numpy()).transform(H)[0][:-1] if ellipse_from_output: # Get ElliFit derived ellipse fits from segmentation mask seg_map_temp = copy.deepcopy(seg_map) seg_map_temp[seg_map_temp == 2] += 1 # Pupil by PartSeg standard is 3 seg_map_temp[seg_map_temp == 1] += 1 # Iris by PartSeg standard is 2 pupilPts, irisPts = getValidPoints(seg_map_temp, isPartSeg=False) if np.sum(seg_map_temp == 3) > 50 and type(pupilPts) is not list: if args.skip_ransac: model_pupil = ElliFit(**{'data': pupilPts}) else: model_pupil = ransac(pupilPts, ElliFit, 15, 40, 5e-3, 15).loop() else: print('Not enough pupil points') model_pupil = type('model', (object, ), {}) model_pupil.model = np.array([-1, -1, -1, -1, -1]) if np.sum(seg_map_temp == 2) > 50 and type(irisPts) is not list: if args.skip_ransac: model_iris = ElliFit(**{'data': irisPts}) else: model_iris = ransac(irisPts, ElliFit, 15, 40, 5e-3, 15).loop() else: print('Not enough iris points') model_iris = type('model', (object, ), {}) model_iris.model = np.array([-1, -1, -1, -1, -1]) model_iris.Phi = np.array([-1, -1, -1, -1, -1]) # iris_fit_error = np.inf pupil_ellipse = np.array(model_pupil.model) iris_ellipse = np.array(model_iris.model) if no_ellipse: pupil_ellipse = np.array([-1, -1, -1, -1, -1]) iris_ellipse = np.array([-1, -1, -1, -1, -1]) return seg_map, latent.cpu().numpy(), pupil_ellipse, iris_ellipse
def get_mask_from_cv2_image(image, model, useGpu=True, pupilOnly=False, includeRawPredict=False, channels=3, trim_pupil=False, isEllseg=False, ellsegPrecision=None, useEllsegEllipseAsMask=False): if useGpu: device = torch.device("cuda") else: device = torch.device("cpu") if not isEllseg: img = image.unsqueeze(1) data = img.to(device) output = model(data) rawpredict = get_predictions(output) predict = rawpredict + 1 # print(np.unique(predict[0].cpu().numpy())) pred_img = 1 - predict[0].cpu().numpy() / channels else: img = np.array(Image.fromarray(image).convert("L")) img = (img - img.mean()) / img.std() img = torch.from_numpy(img).unsqueeze(0).to( ellsegPrecision) # Adds a singleton for channels img = img.unsqueeze(0) img = img.to(device).to(ellsegPrecision) x4, x3, x2, x1, x = model.enc(img) op = model.dec(x4, x3, x2, x1, x) rawpredict = get_predictions(op) plt.imshow(rawpredict[0], cmap="BrBG", alpha=0.3) if useEllsegEllipseAsMask: ellpred = model.elReg(x, 0).view(-1) #i1, i2, i3, i4, i5, p1, p2, p3, p4, p5 = ellpred[0].cpu().detach().numpy() _, _, H, W = img.shape H_mat = np.array([[W / 2, 0, W / 2], [0, H / 2, H / 2], [0, 0, 1]]) #import pdb #pdb.set_trace() i_cx, i_cy, i_a, i_b, i_theta, _ = my_ellipse( ellpred[:5].tolist()).transform(H_mat)[0] p_cx, p_cy, p_a, p_b, p_theta, _ = my_ellipse( ellpred[5:].tolist()).transform(H_mat)[0] ellimage = np.full((int(H), int(W)), 2 / 3) startAngle = 0 endAngle = 360 iris_color = 1 / 3 pupil_color = 0.0 pred_img = draw_ellipse(ellimage, (i_cx, i_cy), (i_a, i_b), i_theta, startAngle, endAngle, iris_color, -1) pred_img = draw_ellipse(ellimage, (p_cx, p_cy), (p_a, p_b), p_theta, startAngle, endAngle, pupil_color, -1) else: predict = rawpredict + 1 pred_img = 1 - predict[0].cpu().numpy() / channels #print(pred_img) # trim pupil if asked to if trim_pupil: newimg = np.invert(pred_img > 0) labeled_img = measure.label(newimg) labels = np.unique(labeled_img) newimg = np.zeros((newimg.shape[0], newimg.shape[1])) old_sum = 0 old_label = None for label in (y for y in labels if y != 0): if np.sum(labeled_img == label) > old_sum: old_sum = np.sum(labeled_img == label) old_label = label if old_label is not None: newimg = newimg + (labeled_img == old_label) newimg[newimg == 0] = 2 newimg[newimg == 1] = 0 newimg[newimg == 2] = 1 pred_img[pred_img == 0] = 1 - (1 / channels) pred_img[newimg == 0] = 0 #print(np.unique(pred_img)) if pupilOnly: pred_img = np.ceil(pred_img) * 0.5 if includeRawPredict: return pred_img, rawpredict return pred_img
def get_pupil_ellipse_from_cv2_image(image, model, useGpu=True, predict=None, isEllseg=False, ellsegPrecision=None, ellsegEllipse=False, debugWindowName=None): """ OUTPUT FORMAT { 0: center x, 1: center y, 2: ellipse major axis radius, 3: ellipse minor axis radius, 4: ellipse angle } """ if useGpu: device = torch.device("cuda") else: device = torch.device("cpu") if predict is None: if not isEllseg: img = image.unsqueeze(1) data = img.to(device) output = model(data) predict = get_predictions(output) pred_img = predict[0].numpy() else: # w:320 h:240 img = np.array(transforms.ToPILImage()(image).convert("L")) img = (img - img.mean()) / img.std() img = torch.from_numpy(img).unsqueeze(0).to( ellsegPrecision) # Adds a singleton for channels img = img.unsqueeze(0) img = img.to(device).to(ellsegPrecision) x4, x3, x2, x1, x = model.enc(img) op = model.dec(x4, x3, x2, x1, x) if ellsegEllipse: # Option to get ellipse directly from ellseg output ellpred = model.elReg(x, 0).view(-1) _, _, H, W = img.shape H_mat = np.array([[W / 2, 0, W / 2], [0, H / 2, H / 2], [0, 0, 1]]) p_cx, p_cy, p_a, p_b, p_theta, _ = my_ellipse( ellpred[5:].tolist()).transform(H_mat)[0] return [p_cx, p_cy, p_a, p_b, p_theta] # [centerX, centerY, axis1, axis2, angle] #elOut = model.elReg(x, 0) # Linear regression to ellipse parameters #print(elOut.shape) predict = get_predictions(op) pred_img = predict[0].numpy() # cv2.imshow("ELLIPSE", pred_img/2) else: pred_img = predict[0].numpy() if debugWindowName is not None: outIm = pred_img / np.max(pred_img) cv2.imshow(debugWindowName, outIm) return get_pupil_parameters(pred_img)
def generateImageGrid(I, mask, elNorm, pupil_center, cond, heatmaps=False, override=False): ''' Parameters ---------- I : numpy array [B, H, W] A batchfirst array which holds images mask : numpy array [B, H, W] A batch first array which holds for individual pixels. hMaps: numpy array [B, C, N, H, W] N is the # of points, C is the category the points belong to (iris or pupil). Heatmaps are gaussians centered around point of interest elNorm:numpy array [B, C, 5] Normalized ellipse parameters pupil_center : numpy array [B, 2] Identified pupil center for plotting. cond : numpy array [B, 5] A flag array which holds information about what information is present. heatmaps : bool, optional Unless specificed, does not show the heatmaps of predicted points override : bool, optional An override flag which plots data despite being demarked in the flag array. Generally used during testing. The default is False. Returns ------- I_o : numpy array [Ho, Wo] Returns an array holding concatenated images from the input overlayed with segmentation mask, pupil center and pupil ellipse. Note: If masks exist, then ellipse parameters would exist aswell. ''' B, H, W = I.shape mesh = create_meshgrid(H, W, normalized_coordinates=True) # 1xHxWx2 H = np.array([[W / 2, 0, W / 2], [0, H / 2, H / 2], [0, 0, 1]]) I_o = [] for i in range(0, min(16, cond.shape[0])): im = I[i, ...].squeeze() - I[i, ...].min() im = cv2.equalizeHist(np.uint8(255 * im / im.max())) im = np.stack([im for i in range(0, 3)], axis=2) if (not cond[i, 1]) or override: # If masks exists rr, cc = np.where(mask[i, ...] == 1) im[rr, cc, ...] = np.array([0, 255, 0]) # Green rr, cc = np.where(mask[i, ...] == 2) im[rr, cc, ...] = np.array([255, 255, 0]) # Yellow # Just for experiments. Please ignore. el_iris = elNorm[i, 0, ...] X = (mesh[..., 0].squeeze() - el_iris[0])*np.cos(el_iris[-1])+\ (mesh[..., 1].squeeze() - el_iris[1])*np.sin(el_iris[-1]) Y = -(mesh[..., 0].squeeze() - el_iris[0])*np.sin(el_iris[-1])+\ (mesh[..., 1].squeeze() - el_iris[1])*np.cos(el_iris[-1]) wtMat = (X / el_iris[2])**2 + (Y / el_iris[3])**2 - 1 # [rr_i, cc_i] = np.where(wtMat< 0) try: el_iris = my_ellipse(elNorm[i, 0, ...]).transform(H)[0] el_pupil = my_ellipse(elNorm[i, 1, ...]).transform(H)[0] except: print( 'Warning: inappropriate ellipses. Defaulting to not break runtime..' ) el_iris = np.array([W / 2, H / 2, W / 8, H / 8, 0.0]).astype(np.float32) el_pupil = np.array([W / 2, H / 2, W / 4, H / 4, 0.0]).astype(np.float32) [rr_i, cc_i] = draw.ellipse_perimeter(int(el_iris[1]), int(el_iris[0]), int(el_iris[3]), int(el_iris[2]), orientation=el_iris[4]) [rr_p, cc_p] = draw.ellipse_perimeter(int(el_pupil[1]), int(el_pupil[0]), int(el_pupil[3]), int(el_pupil[2]), orientation=el_pupil[4]) rr_i = rr_i.clip(6, im.shape[0] - 6) rr_p = rr_p.clip(6, im.shape[0] - 6) cc_i = cc_i.clip(6, im.shape[1] - 6) cc_p = cc_p.clip(6, im.shape[1] - 6) im[rr_i, cc_i, ...] = np.array([0, 0, 255]) im[rr_p, cc_p, ...] = np.array([255, 0, 0]) if (not cond[i, 0]) or override: # If pupil center exists rr, cc = draw.disk((pupil_center[i, 1].clamp(6, im.shape[0] - 6), pupil_center[i, 0].clamp(6, im.shape[1] - 6)), 5) im[rr, cc, ...] = 255 I_o.append(im) I_o = np.stack(I_o, axis=0) I_o = np.moveaxis(I_o, 3, 1) I_o = make_grid(torch.from_numpy(I_o).to(torch.float), nrow=4) I_o = I_o - I_o.min() I_o = I_o / I_o.max() return I_o
Data['Images'].append(I) Data['Masks'].append(mask) Data['Masks_noSkin'].append(mask_noSkin) Data['Info'].append(str_imName) keydict['resolution'].append(I.shape) keydict['archive'].append(ds_name) temp = 255 * (mask_noSkin == 3) edge = cv2.Canny(temp.astype(np.uint8), 10, 150) + cv2.Canny( (255 - temp).astype(np.uint8), 10, 150) r, c = np.where(edge) temp_pts = np.stack([c, r], axis=1) # Improve readability model_pupil = ransac(temp_pts, ElliFit, 15, 10, 0.05, np.round(temp_pts.shape[0] / 2)).loop() pupil_fit_error = my_ellipse(model_pupil.model).verify(temp_pts) pupil_loc = model_pupil.model[:2] temp = 255 * ((mask_noSkin == 2) | (mask_noSkin == 3)) edge = cv2.Canny(temp.astype(np.uint8), 10, 150) + cv2.Canny( (255 - temp).astype(np.uint8), 10, 150) r, c = np.where(edge) temp_pts = np.stack([c, r], axis=1) model_iris = ransac(temp_pts, ElliFit, 15, 10, 0.05, np.round(temp_pts.shape[0] / 2)).loop() iris_fit_error = my_ellipse(model_iris.model).verify(temp_pts) Data['Fits']['pupil'].append(model_pupil.model) Data['Fits']['iris'].append(model_iris.model)
# Ellipse values defined in eucid coordinates (Y axis up) cx = 180 cy = 180 a = 120 b = 60 deg = 30 [rr, cc] = ellipse(cy, cx, b, a, rotation=np.deg2rad(-deg)) I = np.zeros((400, 400)) I[rr, cc] = 1 H = np.array([[2 / 20, 0, -1], [0, 2 / 20, -1], [0, 0, 1]]) e1 = np.array([cx, cy, a, b, np.deg2rad(deg)]) p1 = my_ellipse(e1) e2 = p1.transform(H)[0][:-1] p2 = my_ellipse(e2) e3 = p2.transform(np.linalg.inv(H))[0][:-1] print('Original ellipse: {}'.format(np.round(e1, 4))) print('Normalized ellipse: {}'.format(e2)) print('Recon ellipse: {}'.format(np.round(e3, 4))) pts1 = p1.generatePoints(50, 'equiAngle') pts2 = p2.generatePoints(50, 'equiAngle') N = len(pts1[0]) fit1 = ElliFit(**{'data': np.stack(pts1, 1)}) fit2 = ElliFit(**{'data': np.stack(pts2, 1)})
cone_params = [a,b,c,d,f,g,h,u,v,w] return M, cone_params, eig_vals H5_file = 'riteyes_s-natural_15_4.h5' path_data = '/media/rakshit/Monster/Datasets/All' f = h5py.File(os.path.join(path_data, H5_file), 'r') im_num = 143 mask = f['Masks_noSkin'][im_num, ...] image = f['Images'][im_num, ...] iris_ellipse = f['Fits']['iris'][im_num, ...] f.close() iris_quad = my_ellipse(iris_ellipse.tolist()).quad.tolist() a, hh, b, gg, ff, d = iris_quad h, g, f = hh/2, gg/2, ff/2 ellipse_quad = torch.tensor([a, h, b, g, f, d]) #%% Step 1 - Get conic equation M, cone_params, eig_vals = get_cone_quad(ellipse_quad) #%% Step 2 - Get reduce conic equation # alphas = get_reduced(cone_param)
def getPts(param): ellipse_mod = my_ellipse(param) x_1, y_1 = ellipse_mod.generatePoints(50, 'equiSlope') x_2, y_2 = ellipse_mod.generatePoints(50, 'equiAngle') return np.concatenate([x_1, x_2]), np.concatenate([y_1, y_2])
return patches n_pts = 5 I_res = (240, 240) H = np.array([[2/240, 0, -1], [0, 2/240, -1], [0, 0, 1]]) angleItr = np.linspace(10, 170, n_pts) eccItr = np.linspace(0.4, 1.6, n_pts) + 1e-5 # Never a perfect 1 data = {'I':[], 'xPts':[], 'yPts':[]} for ang in np.nditer(angleItr): for ecc in np.nditer(eccItr): # Input as param param = np.array([130, 110, 40, 40*ecc, np.deg2rad(ang)]) #print('Phi: {}'.format(np.round(my_ellipse(param).recover_Phi(), 2))) param_norm = my_ellipse(param).transform(H)[0][:5] print('Theta: {}. E: {}'.format(ang, ecc)) print('Param: {}'.format(np.round(param, 2))) print('Param norm: {}'.format(np.round(param_norm, 2))) print('Phi: {}'.format(np.round(my_ellipse(param_norm).recover_Phi(), 2))) x_pts, y_pts = getPts(param) I = np.zeros(I_res) rr, cc = ellipse(param[1], param[0], param[3], param[2], shape=I_res, rotation=param[4]) I[rr, cc] = 1 data['I'].append(I) data['xPts'].append(x_pts) data['yPts'].append(y_pts) data['I'] = np.stack(data['I'], axis=0) data['xPts'] = np.stack(data['xPts'], axis=0) data['yPts'] = np.stack(data['yPts'], axis=0)