def detect(self,frame,user_roi,visualize=False):
u_r = user_roi
if self.window_should_open:
self.open_window((frame.img.shape[1],frame.img.shape[0]))
if self.window_should_close:
self.close_window()
if self._window:
debug_img = np.zeros(frame.img.shape,frame.img.dtype)
#get the user_roi
img = frame.img
r_img = img[u_r.view]
# bias_field = preproc.EstimateBias(r_img)
# r_img = preproc.Unbias(r_img, bias_field)
r_img = preproc.GaussBlur(r_img)
r_img = preproc.RobustRescale(r_img)
frame.img[u_r.view] = r_img
gray_img = cv2.cvtColor(r_img,cv2.COLOR_BGR2GRAY)
# coarse pupil detection
if self.coarse_detection.value:
integral = cv2.integral(gray_img)
integral = np.array(integral,dtype=c_float)
x,y,w,response = eye_filter(integral,self.coarse_filter_min,self.coarse_filter_max)
p_r = Roi(gray_img.shape)
if w>0:
p_r.set((y,x,y+w,x+w))
else:
p_r.set((0,0,-1,-1))
else:
p_r = Roi(gray_img.shape)
p_r.set((0,0,None,None))
w = img.shape[0]/2
coarse_pupil_width = w/2.
padding = coarse_pupil_width/4.
pupil_img = gray_img[p_r.view]
# binary thresholding of pupil dark areas
hist = cv2.calcHist([pupil_img],[0],None,[256],[0,256]) #(images, channels, mask, histSize, ranges[, hist[, accumulate]])
bins = np.arange(hist.shape[0])
spikes = bins[hist[:,0]>40] # every intensity seen in more than 40 pixels
if spikes.shape[0] >0:
lowest_spike = spikes.min()
highest_spike = spikes.max()
else:
lowest_spike = 200
highest_spike = 255
offset = self.intensity_range.value
spectral_offset = 5
if visualize:
# display the histogram
sx,sy = 100,1
colors = ((0,0,255),(255,0,0),(255,255,0),(255,255,255))
h,w,chan = img.shape
hist *= 1./hist.max() # normalize for display
for i,h in zip(bins,hist[:,0]):
c = colors[1]
cv2.line(img,(w,int(i*sy)),(w-int(h*sx),int(i*sy)),c)
cv2.line(img,(w,int(lowest_spike*sy)),(int(w-.5*sx),int(lowest_spike*sy)),colors[0])
cv2.line(img,(w,int((lowest_spike+offset)*sy)),(int(w-.5*sx),int((lowest_spike+offset)*sy)),colors[2])
cv2.line(img,(w,int((highest_spike)*sy)),(int(w-.5*sx),int((highest_spike)*sy)),colors[0])
cv2.line(img,(w,int((highest_spike- spectral_offset )*sy)),(int(w-.5*sx),int((highest_spike - spectral_offset)*sy)),colors[3])
# create dark and spectral glint masks
self.bin_thresh.value = lowest_spike
binary_img = bin_thresholding(pupil_img,image_upper=lowest_spike + offset)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7,7))
cv2.dilate(binary_img, kernel,binary_img, iterations=2)
spec_mask = bin_thresholding(pupil_img, image_upper=highest_spike - spectral_offset)
cv2.erode(spec_mask, kernel,spec_mask, iterations=1)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9,9))
#open operation to remove eye lashes
pupil_img = cv2.morphologyEx(pupil_img, cv2.MORPH_OPEN, kernel)
if self.blur > 1:
pupil_img = cv2.medianBlur(pupil_img,self.blur.value)
edges = cv2.Canny(pupil_img,
self.canny_thresh,
self.canny_thresh*self.canny_ratio,
apertureSize= self.canny_aperture)
# remove edges in areas not dark enough and where the glint is (spectral refelction from IR leds)
edges = cv2.min(edges, spec_mask)
edges = cv2.min(edges,binary_img)
overlay = img[u_r.view][p_r.view]
if visualize:
b,g,r = overlay[:,:,0],overlay[:,:,1],overlay[:,:,2]
g[:] = cv2.max(g,edges)
b[:] = cv2.max(b,binary_img)
b[:] = cv2.min(b,spec_mask)
# draw a frame around the automatic pupil ROI in overlay.
overlay[::2,0] = 255 #yeay numpy broadcasting
overlay[::2,-1]= 255
overlay[0,::2] = 255
overlay[-1,::2]= 255
# draw a frame around the area we require the pupil center to be.
overlay[padding:-padding:4,padding] = 255
overlay[padding:-padding:4,-padding]= 255
overlay[padding,padding:-padding:4] = 255
overlay[-padding,padding:-padding:4]= 255
if visualize:
c = (100.,frame.img.shape[0]-100.)
e_max = ((c),(self.pupil_max.value,self.pupil_max.value),0)
e_recent = ((c),(self.target_size.value,self.target_size.value),0)
e_min = ((c),(self.pupil_min.value,self.pupil_min.value),0)
cv2.ellipse(frame.img,e_min,(0,0,255),1)
cv2.ellipse(frame.img,e_recent,(0,255,0),1)
cv2.ellipse(frame.img,e_max,(0,0,255),1)
#get raw edge pix for later
raw_edges = cv2.findNonZero(edges)
def ellipse_true_support(e,raw_edges):
a,b = e[1][0]/2.,e[1][1]/2. # major minor radii of candidate ellipse
ellipse_circumference = np.pi*abs(3*(a+b)-np.sqrt(10*a*b+3*(a**2+b**2)))
distances = dist_pts_ellipse(e,raw_edges)
support_pixels = raw_edges[distances<=1.3]
# support_ratio = support_pixel.shape[0]/ellipse_circumference
return support_pixels,ellipse_circumference
# if we had a good ellipse before ,let see if it is still a good first guess:
if self.strong_prior:
e = p_r.sub_vector(u_r.sub_vector(self.strong_prior[0])),self.strong_prior[1],self.strong_prior[2]
self.strong_prior = None
if raw_edges is not None:
support_pixels,ellipse_circumference = ellipse_true_support(e,raw_edges)
support_ratio = support_pixels.shape[0]/ellipse_circumference
if support_ratio >= self.strong_perimeter_ratio_range[0]:
refit_e = cv2.fitEllipse(support_pixels)
if self._window:
cv2.ellipse(debug_img,e,(255,100,100),thickness=4)
cv2.ellipse(debug_img,refit_e,(0,0,255),thickness=1)
e = refit_e
self.strong_prior = u_r.add_vector(p_r.add_vector(e[0])),e[1],e[2]
goodness = min(1.,support_ratio)
pupil_ellipse = {}
pupil_ellipse['confidence'] = goodness
pupil_ellipse['ellipse'] = e
pupil_ellipse['roi_center'] = e[0]
pupil_ellipse['major'] = max(e[1])
pupil_ellipse['minor'] = min(e[1])
pupil_ellipse['apparent_pupil_size'] = max(e[1])
pupil_ellipse['axes'] = e[1]
pupil_ellipse['angle'] = e[2]
e_img_center =u_r.add_vector(p_r.add_vector(e[0]))
norm_center = normalize(e_img_center,(frame.img.shape[1], frame.img.shape[0]),flip_y=True)
pupil_ellipse['norm_pupil'] = norm_center
pupil_ellipse['center'] = e_img_center
pupil_ellipse['timestamp'] = frame.timestamp
self.target_size.value = max(e[1])
self.confidence.value = goodness
self.confidence_hist.append(goodness)
self.confidence_hist[:-200]=[]
if self._window:
#draw a little animation of confidence
cv2.putText(debug_img, 'good',(410,debug_img.shape[0]-100), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100))
cv2.putText(debug_img, 'threshold',(410,debug_img.shape[0]-int(self.final_perimeter_ratio_range[0]*100)), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100))
cv2.putText(debug_img, 'no detection',(410,debug_img.shape[0]-10), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100))
lines = np.array([[[2*x,debug_img.shape[0]-int(100*y)],[2*x,debug_img.shape[0]]] for x,y in enumerate(self.confidence_hist)])
cv2.polylines(debug_img,lines,isClosed=False,color=(255,100,100))
self.gl_display_in_window(debug_img)
return pupil_ellipse
# from edges to contours
contours, hierarchy = cv2.findContours(edges,
mode=cv2.RETR_LIST,
method=cv2.CHAIN_APPROX_NONE,offset=(0,0)) #TC89_KCOS
# contours is a list containing array([[[108, 290]],[[111, 290]]], dtype=int32) shape=(number of points,1,dimension(2) )
### first we want to filter out the bad stuff
# to short
good_contours = [c for c in contours if c.shape[0]>self.min_contour_size.value]
# now we learn things about each contour through looking at the curvature.
# For this we need to simplyfy the contour so that pt to pt angles become more meaningfull
aprox_contours = [cv2.approxPolyDP(c,epsilon=1.5,closed=False) for c in good_contours]
if self._window:
x_shift = coarse_pupil_width*2
color = zip(range(0,250,15),range(0,255,15)[::-1],range(230,250))
split_contours = []
for c in aprox_contours:
curvature = GetAnglesPolyline(c)
# we split whenever there is a real kink (abs(curvature)<right angle) or a change in the genreal direction
kink_idx = find_kink_and_dir_change(curvature,80)
segs = split_at_corner_index(c,kink_idx)
#TODO: split at shart inward turns
for s in segs:
if s.shape[0]>2:
split_contours.append(s)
if self._window:
c = color.pop(0)
color.append(c)
s = s.copy()
s[:,:,0] += debug_img.shape[1]-coarse_pupil_width*2
# s[:,:,0] += x_shift
# x_shift += 5
cv2.polylines(debug_img,[s],isClosed=False,color=map(lambda x: x,c),thickness = 1,lineType=4)#cv2.CV_AA
split_contours.sort(key=lambda x:-x.shape[0])
# print [x.shape[0]for x in split_contours]
if len(split_contours) == 0:
# not a single usefull segment found -> no pupil found
self.confidence.value = 0
self.confidence_hist.append(0)
if self._window:
self.gl_display_in_window(debug_img)
return {'timestamp':frame.timestamp,'norm_pupil':None}
# removing stubs makes combinatorial search feasable
split_contours = [c for c in split_contours if c.shape[0]>3]
def ellipse_filter(e):
in_center = padding < e[0][1] < pupil_img.shape[0]-padding and padding < e[0][0] < pupil_img.shape[1]-padding
if in_center:
is_round = min(e[1])/max(e[1]) >= self.min_ratio
if is_round:
right_size = self.pupil_min.value <= max(e[1]) <= self.pupil_max.value
if right_size:
return True
return False
def ellipse_on_blue(e):
center_on_dark = binary_img[e[0][1],e[0][0]]
return bool(center_on_dark)
def ellipse_support_ratio(e,contours):
a,b = e[1][0]/2.,e[1][1]/2. # major minor radii of candidate ellipse
ellipse_area = np.pi*a*b
ellipse_circumference = np.pi*abs(3*(a+b)-np.sqrt(10*a*b+3*(a**2+b**2)))
actual_area = cv2.contourArea(cv2.convexHull(np.concatenate(contours)))
actual_contour_length = sum([cv2.arcLength(c,closed=False) for c in contours])
area_ratio = actual_area / ellipse_area
perimeter_ratio = actual_contour_length / ellipse_circumference #we assume here that the contour lies close to the ellipse boundary
return perimeter_ratio,area_ratio
def final_fitting(c,edges):
#use the real edge pixels to fit, not the aproximated contours
support_mask = np.zeros(edges.shape,edges.dtype)
cv2.polylines(support_mask,c,isClosed=False,color=(255,255,255),thickness=2)
# #draw into the suport mast with thickness 2
new_edges = cv2.min(edges, support_mask)
new_contours = cv2.findNonZero(new_edges)
if self._window:
new_edges[new_edges!=0] = 255
overlay[:,:,1] = cv2.max(overlay[:,:,1], new_edges)
overlay[:,:,2] = cv2.max(overlay[:,:,2], new_edges)
overlay[:,:,2] = cv2.max(overlay[:,:,2], new_edges)
new_e = cv2.fitEllipse(new_contours)
return new_e,new_contours
# finding poential candidates for ellipse seeds that describe the pupil.
strong_seed_contours = []
weak_seed_contours = []
for idx, c in enumerate(split_contours):
if c.shape[0] >=5:
e = cv2.fitEllipse(c)
# is this ellipse a plausible canditate for a pupil?
if ellipse_filter(e):
distances = dist_pts_ellipse(e,c)
fit_variance = np.sum(distances**2)/float(distances.shape[0])
if fit_variance <= self.inital_ellipse_fit_threshhold:
# how much ellipse is supported by this contour?
perimeter_ratio,area_ratio = ellipse_support_ratio(e,[c])
# logger.debug('Ellipse no %s with perimeter_ratio: %s , area_ratio: %s'%(idx,perimeter_ratio,area_ratio))
if self.strong_perimeter_ratio_range[0]<= perimeter_ratio <= self.strong_perimeter_ratio_range[1] and self.strong_area_ratio_range[0]<= area_ratio <= self.strong_area_ratio_range[1]:
strong_seed_contours.append(idx)
if self._window:
cv2.polylines(debug_img,[c],isClosed=False,color=(255,100,100),thickness=4)
e = (e[0][0]+debug_img.shape[1]-coarse_pupil_width*4,e[0][1]),e[1],e[2]
cv2.ellipse(debug_img,e,color=(255,100,100),thickness=3)
else:
weak_seed_contours.append(idx)
if self._window:
cv2.polylines(debug_img,[c],isClosed=False,color=(255,0,0),thickness=2)
e = (e[0][0]+debug_img.shape[1]-coarse_pupil_width*4,e[0][1]),e[1],e[2]
cv2.ellipse(debug_img,e,color=(255,0,0))
sc = np.array(split_contours)
if strong_seed_contours:
seed_idx = strong_seed_contours
elif weak_seed_contours:
seed_idx = weak_seed_contours
if not (strong_seed_contours or weak_seed_contours):
if self._window:
self.gl_display_in_window(debug_img)
self.confidence.value = 0
self.confidence_hist.append(0)
return {'timestamp':frame.timestamp,'norm_pupil':None}
# if self._window:
# cv2.polylines(debug_img,[split_contours[i] for i in seed_idx],isClosed=False,color=(255,255,100),thickness=3)
def ellipse_eval(contours):
c = np.concatenate(contours)
e = cv2.fitEllipse(c)
d = dist_pts_ellipse(e,c)
fit_variance = np.sum(d**2)/float(d.shape[0])
return fit_variance <= self.inital_ellipse_fit_threshhold
solutions = pruning_quick_combine(split_contours,ellipse_eval,seed_idx,max_evals=1000,max_depth=5)
solutions = filter_subsets(solutions)
ratings = []
for s in solutions:
e = cv2.fitEllipse(np.concatenate(sc[s]))
if self._window:
cv2.ellipse(debug_img,e,(0,150,100))
support_pixels,ellipse_circumference = ellipse_true_support(e,raw_edges)
support_ratio = support_pixels.shape[0]/ellipse_circumference
# TODO: refine the selection of final canditate
if support_ratio >=self.final_perimeter_ratio_range[0] and ellipse_filter(e):
ratings.append(support_pixels.shape[0])
if support_ratio >=self.strong_perimeter_ratio_range[0]:
self.strong_prior = u_r.add_vector(p_r.add_vector(e[0])),e[1],e[2]
if self._window:
cv2.ellipse(debug_img,e,(0,255,255),thickness = 2)
else:
#not a valid solution, bad rating
ratings.append(-1)
# selected ellipse
if max(ratings) == -1:
#no good final ellipse found
if self._window:
self.gl_display_in_window(debug_img)
self.confidence.value = 0
self.confidence_hist.append(0)
return {'timestamp':frame.timestamp,'norm_pupil':None}
best = solutions[ratings.index(max(ratings))]
e = cv2.fitEllipse(np.concatenate(sc[best]))
#final calculation of goodness of fit
support_pixels,ellipse_circumference = ellipse_true_support(e,raw_edges)
support_ratio = support_pixels.shape[0]/ellipse_circumference
goodness = min(1.,support_ratio)
#final fitting and return of result
new_e,final_edges = final_fitting(sc[best],edges)
size_dif = abs(1 - max(e[1])/max(new_e[1]))
if ellipse_filter(new_e) and size_dif < .3:
if self._window:
cv2.ellipse(debug_img,new_e,(0,255,0))
e = new_e
pupil_ellipse = {}
pupil_ellipse['confidence'] = goodness
pupil_ellipse['ellipse'] = e
pupil_ellipse['pos_in_roi'] = e[0]
pupil_ellipse['major'] = max(e[1])
pupil_ellipse['apparent_pupil_size'] = max(e[1])
pupil_ellipse['minor'] = min(e[1])
pupil_ellipse['axes'] = e[1]
pupil_ellipse['angle'] = e[2]
e_img_center =u_r.add_vector(p_r.add_vector(e[0]))
norm_center = normalize(e_img_center,(frame.img.shape[1], frame.img.shape[0]),flip_y=True)
pupil_ellipse['norm_pupil'] = norm_center
pupil_ellipse['center'] = e_img_center
pupil_ellipse['timestamp'] = frame.timestamp
self.target_size.value = max(e[1])
self.confidence.value = goodness
self.confidence_hist.append(goodness)
self.confidence_hist[:-200]=[]
if self._window:
#draw a little animation of confidence
cv2.putText(debug_img, 'good',(410,debug_img.shape[0]-100), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100))
cv2.putText(debug_img, 'threshold',(410,debug_img.shape[0]-int(self.final_perimeter_ratio_range[0]*100)), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100))
cv2.putText(debug_img, 'no detection',(410,debug_img.shape[0]-10), cv2.FONT_HERSHEY_SIMPLEX,0.3,(255,100,100))
lines = np.array([[[2*x,debug_img.shape[0]-int(100*y)],[2*x,debug_img.shape[0]]] for x,y in enumerate(self.confidence_hist)])
cv2.polylines(debug_img,lines,isClosed=False,color=(255,100,100))
self.gl_display_in_window(debug_img)
return pupil_ellipse
def detect(self, frame, u_roi, visualize=False):
#get the user_roi
img = frame.img
r_img = img[u_roi.lY:u_roi.uY, u_roi.lX:u_roi.uX]
gray_img = grayscale(r_img)
# coarse pupil detection
integral = cv2.integral(gray_img)
integral = np.array(integral, dtype=c_float)
x, y, w, response = eye_filter(integral, 100, 400)
p_roi = Roi(gray_img.shape)
if w > 0:
p_roi.set((y, x, y + w, x + w))
else:
p_roi.set((0, 0, -1, -1))
coarse_pupil_center = x + w / 2., y + w / 2.
coarse_pupil_width = w / 2.
padding = coarse_pupil_width / 4.
pupil_img = gray_img[p_roi.lY:p_roi.uY, p_roi.lX:p_roi.uX]
# binary thresholding of pupil dark areas
hist = cv2.calcHist([pupil_img], [0], None, [256], [
0, 256
]) #(images, channels, mask, histSize, ranges[, hist[, accumulate]])
bins = np.arange(hist.shape[0])
spikes = bins[hist[:, 0] >
40] # every intensity seen in more than 40 pixels
if spikes.shape[0] > 0:
lowest_spike = spikes.min()
highest_spike = spikes.max()
else:
lowest_spike = 200
highest_spike = 255
offset = self.intensity_range.value
spectral_offset = 5
if visualize:
# display the histogram
sx, sy = 100, 1
colors = ((0, 0, 255), (255, 0, 0), (255, 255, 0), (255, 255, 255))
h, w, chan = img.shape
hist *= 1. / hist.max() # normalize for display
for i, h in zip(bins, hist[:, 0]):
c = colors[1]
cv2.line(img, (w, int(i * sy)), (w - int(h * sx), int(i * sy)),
c)
cv2.line(img, (w, int(lowest_spike * sy)),
(int(w - .5 * sx), int(lowest_spike * sy)), colors[0])
cv2.line(img, (w, int((lowest_spike + offset) * sy)),
(int(w - .5 * sx), int(
(lowest_spike + offset) * sy)), colors[2])
cv2.line(img, (w, int((highest_spike) * sy)),
(int(w - .5 * sx), int((highest_spike) * sy)), colors[0])
cv2.line(
img, (w, int((highest_spike - spectral_offset) * sy)),
(int(w - .5 * sx), int(
(highest_spike - spectral_offset) * sy)), colors[3])
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9, 9))
#open operation to remove eye lashes
pupil_img = cv2.morphologyEx(pupil_img, cv2.MORPH_OPEN, kernel)
# PARAMS = {}
# blob_detector = cv2.SimpleBlobDetector(**PARAMS)
# kps = blob_detector.detect(pupil_img)
# blur = cv2.GaussianBlur(pupil_img,(5,5),0)
blur = pupil_img
# ret3,th3 = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
ret3, th3 = cv2.threshold(blur, lowest_spike + offset, 255,
cv2.THRESH_BINARY)
# ret3,th3 = cv2.threshold(th3,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
th3 = cv2.Laplacian(th3, cv2.CV_64F)
edges = cv2.Canny(pupil_img,
self.canny_thresh.value,
self.canny_thresh.value * self.canny_ratio.value,
apertureSize=self.canny_aperture.value)
r_img[p_roi.lY:p_roi.uY, p_roi.lX:p_roi.uX, 1] = th3
r_img[p_roi.lY:p_roi.uY, p_roi.lX:p_roi.uX, 2] = edges
# for kp in kps:
# print kp.pt
# cv2.circle(r_img[p_roi.lY:p_roi.uY,p_roi.lX:p_roi.uX],tuple(map(int,kp.pt,)),10,(255,255,255))
no_result = {}
no_result['timestamp'] = frame.timestamp
no_result['norm_pupil'] = None
return no_result
def detect(self, frame, u_roi, visualize=False):
if self.window_should_open:
self.open_window()
if self.window_should_close:
self.close_window()
#get the user_roi
img = frame.img
r_img = img[u_roi.lY:u_roi.uY, u_roi.lX:u_roi.uX]
gray_img = grayscale(r_img)
# coarse pupil detection
integral = cv2.integral(gray_img)
integral = np.array(integral, dtype=c_float)
x, y, w, response = eye_filter(integral, self.coarse_filter_min,
self.coarse_filter_max)
p_roi = Roi(gray_img.shape)
if w > 0:
p_roi.set((y, x, y + w, x + w))
else:
p_roi.set((0, 0, -1, -1))
coarse_pupil_center = x + w / 2., y + w / 2.
coarse_pupil_width = w / 2.
padding = coarse_pupil_width / 4.
pupil_img = gray_img[p_roi.lY:p_roi.uY, p_roi.lX:p_roi.uX]
# binary thresholding of pupil dark areas
hist = cv2.calcHist([pupil_img], [0], None, [256], [
0, 256
]) #(images, channels, mask, histSize, ranges[, hist[, accumulate]])
bins = np.arange(hist.shape[0])
spikes = bins[hist[:, 0] >
40] # every intensity seen in more than 40 pixels
if spikes.shape[0] > 0:
lowest_spike = spikes.min()
highest_spike = spikes.max()
else:
lowest_spike = 200
highest_spike = 255
offset = self.intensity_range.value
spectral_offset = 5
if visualize:
# display the histogram
sx, sy = 100, 1
colors = ((0, 0, 255), (255, 0, 0), (255, 255, 0), (255, 255, 255))
h, w, chan = img.shape
hist *= 1. / hist.max() # normalize for display
for i, h in zip(bins, hist[:, 0]):
c = colors[1]
cv2.line(img, (w, int(i * sy)), (w - int(h * sx), int(i * sy)),
c)
cv2.line(img, (w, int(lowest_spike * sy)),
(int(w - .5 * sx), int(lowest_spike * sy)), colors[0])
cv2.line(img, (w, int((lowest_spike + offset) * sy)),
(int(w - .5 * sx), int(
(lowest_spike + offset) * sy)), colors[2])
cv2.line(img, (w, int((highest_spike) * sy)),
(int(w - .5 * sx), int((highest_spike) * sy)), colors[0])
cv2.line(
img, (w, int((highest_spike - spectral_offset) * sy)),
(int(w - .5 * sx), int(
(highest_spike - spectral_offset) * sy)), colors[3])
# create dark and spectral glint masks
self.bin_thresh.value = lowest_spike
binary_img = bin_thresholding(pupil_img,
image_upper=lowest_spike + offset)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))
cv2.dilate(binary_img, kernel, binary_img, iterations=2)
spec_mask = bin_thresholding(pupil_img,
image_upper=highest_spike -
spectral_offset)
cv2.erode(spec_mask, kernel, spec_mask, iterations=1)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9, 9))
#open operation to remove eye lashes
pupil_img = cv2.morphologyEx(pupil_img, cv2.MORPH_OPEN, kernel)
if self.blur.value > 1:
pupil_img = cv2.medianBlur(pupil_img, self.blur.value)
edges = cv2.Canny(pupil_img,
self.canny_thresh.value,
self.canny_thresh.value * self.canny_ratio.value,
apertureSize=self.canny_aperture.value)
# edges = cv2.adaptiveThreshold(pupil_img,255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, self.canny_aperture.value, 7)
# remove edges in areas not dark enough and where the glint is (spectral refelction from IR leds)
edges = cv2.min(edges, spec_mask)
edges = cv2.min(edges, binary_img)
if visualize:
overlay = img[u_roi.lY:u_roi.uY,
u_roi.lX:u_roi.uX][p_roi.lY:p_roi.uY,
p_roi.lX:p_roi.uX]
chn_img = grayscale(overlay)
overlay[:, :, 2] = cv2.max(chn_img, edges) #b channel
overlay[:, :, 0] = cv2.max(chn_img, binary_img) #g channel
overlay[:, :, 1] = cv2.min(chn_img, spec_mask) #b channel
pupil_img = frame.img[u_roi.lY:u_roi.uY,
u_roi.lX:u_roi.uX][p_roi.lY:p_roi.uY,
p_roi.lX:p_roi.uX]
# draw a frame around the automatic pupil ROI in overlay...
pupil_img[::2, 0] = 255, 255, 255
pupil_img[::2, -1] = 255, 255, 255
pupil_img[0, ::2] = 255, 255, 255
pupil_img[-1, ::2] = 255, 255, 255
pupil_img[::2, padding] = 255, 255, 255
pupil_img[::2, -padding] = 255, 255, 255
pupil_img[padding, ::2] = 255, 255, 255
pupil_img[-padding, ::2] = 255, 255, 255
frame.img[u_roi.lY:u_roi.uY,
u_roi.lX:u_roi.uX][p_roi.lY:p_roi.uY,
p_roi.lX:p_roi.uX] = pupil_img
# from edges to contours
contours, hierarchy = cv2.findContours(edges,
mode=cv2.RETR_LIST,
method=cv2.CHAIN_APPROX_NONE,
offset=(0, 0)) #TC89_KCOS
# contours is a list containing array([[[108, 290]],[[111, 290]]], dtype=int32) shape=(number of points,1,dimension(2) )
### first we want to filter out the bad stuff
# to short
good_contours = [
c for c in contours if c.shape[0] > self.min_contour_size
]
# now we learn things about each contour though looking at the curvature. For this we need to simplyfy the contour
arprox_contours = [
cv2.approxPolyDP(c, epsilon=1.5, closed=False)
for c in good_contours
]
# cv2.drawContours(pupil_img,good_contours,-1,(255,255,0))
# cv2.drawContours(pupil_img,arprox_contours,-1,(0,0,255))
if self._window:
debug_img = np.zeros(img.shape, img.dtype)
x_shift = coarse_pupil_width * 2 #just vor display
color = zip(range(0, 250, 30),
range(0, 255, 30)[::-1], range(230, 250))
split_contours = []
for c in arprox_contours:
curvature = GetAnglesPolyline(c)
# print curvature
# we split whenever there is a real kink (abs(curvature)<right angle) or a change in the genreal direction
kink_idx = find_kink_and_dir_change(curvature, 100)
# kinks,k_index = convexity_defect(c,curvature)
# print "kink_idx", kink_idx
segs = split_at_corner_index(c, kink_idx)
# print len(segs)
# segs.sort(key=lambda e:-len(e))
for s in segs:
split_contours.append(s)
if self._window:
c = color.pop(0)
color.append(c)
# if s.shape[0] >=5:
# cv2.polylines(debug_img,[s],isClosed=False,color=c)
s = s.copy()
s[:, :, 1] += coarse_pupil_width * 2
cv2.polylines(debug_img, [s], isClosed=False, color=c)
s[:, :, 0] += x_shift
x_shift += 5
cv2.polylines(debug_img, [s], isClosed=False, color=c)
# return {'timestamp':frame.timestamp,'norm_pupil':None}
#these segments may now be smaller, we need to get rid of those not long enough for ellipse fitting
good_contours = [c for c in split_contours if c.shape[0] >= 5]
# cv2.polylines(img,good_contours,isClosed=False,color=(255,255,0))
shape = edges.shape
ellipses = ((cv2.fitEllipse(c), c) for c in good_contours)
ellipses = ((e, c) for e, c in ellipses
if (padding < e[0][1] < shape[0] -
padding and padding < e[0][0] < shape[1] - padding)
) # center is close to roi center
ellipses = ((e, c) for e, c in ellipses
if binary_img[e[0][1],
e[0][0]]) # center is on a dark pixel
ellipses = [(e, c) for e, c in ellipses
if is_round(e, self.target_ratio)] # roundness test
result = []
for e, c in ellipses:
size_dif = size_deviation(e, self.target_size.value)
pupil_ellipse = {}
pupil_ellipse['contour'] = c
a, b = e[1][0] / 2., e[1][
1] / 2. # majar minor radii of candidate ellipse
pupil_ellipse['circumference'] = np.pi * abs(
3 * (a + b) - np.sqrt(10 * a * b + 3 * (a**2 + b**2)))
# pupil_ellipse['convex_hull'] = cv2.convexHull(pupil_ellipse['contour'])
pupil_ellipse['contour_area'] = cv2.contourArea(cv2.convexHull(c))
pupil_ellipse['ellipse_area'] = np.pi * a * b
# print abs(pupil_ellipse['contour_area']-pupil_ellipse['ellipse_area'])
if abs(pupil_ellipse['contour_area'] -
pupil_ellipse['ellipse_area']) < 10:
pupil_ellipse['goodness'] = abs(
pupil_ellipse['contour_area'] -
pupil_ellipse['ellipse_area']
) / 10 #perfect match we'll take this one
else:
pupil_ellipse['goodness'] = size_dif
if visualize:
pass
# cv2.drawContours(pupil_img,[cv2.convexHull(c)],-1,(size_dif,size_dif,255))
# cv2.drawContours(pupil_img,[c],-1,(size_dif,size_dif,255))
pupil_ellipse['pupil_center'] = e[0] # compensate for roi offsets
pupil_ellipse['center'] = u_roi.add_vector(p_roi.add_vector(
e[0])) # compensate for roi offsets
pupil_ellipse['angle'] = e[-1]
pupil_ellipse['axes'] = e[1]
pupil_ellipse['major'] = max(e[1])
pupil_ellipse['minor'] = min(e[1])
pupil_ellipse[
'ratio'] = pupil_ellipse['minor'] / pupil_ellipse['major']
pupil_ellipse['norm_pupil'] = normalize(
pupil_ellipse['center'], (img.shape[1], img.shape[0]),
flip_y=True)
pupil_ellipse['timestamp'] = frame.timestamp
result.append(pupil_ellipse)
#### adding support
if result:
result.sort(key=lambda e: e['goodness'])
# for now we assume that this contour is part of the pupil
the_one = result[0]
# (center, size, angle) = cv2.fitEllipse(the_one['contour'])
# print "itself"
distances = dist_pts_ellipse(cv2.fitEllipse(the_one['contour']),
the_one['contour'])
# print np.average(distances)
# print np.sum(distances)/float(distances.shape[0])
# print "other"
# if self._window:
# cv2.polylines(debug_img,[result[-1]['contour']],isClosed=False,color=(255,255,255),thickness=3)
with_another = np.concatenate(
(result[-1]['contour'], the_one['contour']))
distances = dist_pts_ellipse(cv2.fitEllipse(with_another),
with_another)
# if 1.5 > np.sum(distances)/float(distances.shape[0]):
# if self._window:
# cv2.polylines(debug_img,[result[-1]['contour']],isClosed=False,color=(255,255,255),thickness=3)
perimeter_ratio = cv2.arcLength(
the_one["contour"], closed=False) / the_one['circumference']
if perimeter_ratio > .9:
size_thresh = 0
eccentricity_thresh = 0
elif perimeter_ratio > .5:
size_thresh = the_one['major'] / (5.)
eccentricity_thresh = the_one['major'] / 2.
self.should_sleep = True
else:
size_thresh = the_one['major'] / (3.)
eccentricity_thresh = the_one['major'] / 2.
self.should_sleep = True
if self._window:
center = np.uint16(np.around(the_one['pupil_center']))
cv2.circle(debug_img, tuple(center), int(eccentricity_thresh),
(0, 255, 0), 1)
if self._window:
cv2.polylines(debug_img, [the_one["contour"]],
isClosed=False,
color=(255, 0, 0),
thickness=2)
s = the_one["contour"].copy()
s[:, :, 0] += coarse_pupil_width * 2
cv2.polylines(debug_img, [s],
isClosed=False,
color=(255, 0, 0),
thickness=2)
# but are there other segments that could be used for support?
new_support = [
the_one['contour'],
]
if len(result) > 1:
the_one = result[0]
target_axes = the_one['axes'][0]
# target_mean_curv = np.mean(curvature(the_one['contour'])
for e in result:
# with_another = np.concatenate((e['contour'],the_one['contour']))
# with_another = np.concatenate([r['contour'] for r in result])
with_another = e['contour']
distances = dist_pts_ellipse(cv2.fitEllipse(with_another),
with_another)
# print np.std(distances)
thick = int(np.std(distances))
if 1.5 > np.average(distances) or 1:
if self._window:
# print thick
thick = min(20, thick)
cv2.polylines(debug_img, [e['contour']],
isClosed=False,
color=(255, 255, 255),
thickness=thick)
if self._window:
cv2.polylines(debug_img, [e["contour"]],
isClosed=False,
color=(0, 100, 100))
center_dist = cv2.arcLength(np.array(
[the_one["pupil_center"], e['pupil_center']],
dtype=np.int32),
closed=False)
size_dif = abs(the_one['major'] - e['major'])
# #lets make sure the countour is not behind the_one/'s coutour
# center_point = np.uint16(np.around(the_one['pupil_center']))
# other_center_point = np.uint16(np.around(e['pupil_center']))
# mid_point = the_one["contour"][the_one["contour"].shape[0]/2][0]
# other_mid_point = e["contour"][e["contour"].shape[0]/2][0]
# #reflect around mid_point
# p = center_point - mid_point
# p = np.array((-p[1],-p[0]))
# mir_center_point = p + mid_point
# dist_mid = cv2.arcLength(np.array([mid_point,other_mid_point]),closed=False)
# dist_center = cv2.arcLength(np.array([center_point,other_mid_point]),closed=False)
# if self._window:
# cv2.circle(debug_img,tuple(center_point),3,(0,255,0),2)
# cv2.circle(debug_img,tuple(other_center_point),2,(0,0,255),1)
# # cv2.circle(debug_img,tuple(mir_center_point),3,(0,255,0),2)
# # cv2.circle(debug_img,tuple(mid_point),2,(0,255,0),1)
# # cv2.circle(debug_img,tuple(other_mid_point),2,(0,0,255),1)
# cv2.polylines(debug_img,[np.array([center_point,other_mid_point]),np.array([mid_point,other_mid_point])],isClosed=False,color=(0,255,0))
if center_dist < eccentricity_thresh:
# print dist_mid-dist_center
# if dist_mid > dist_center-20:
if size_dif < size_thresh:
new_support.append(e["contour"])
if self._window:
cv2.polylines(debug_img, [s],
isClosed=False,
color=(255, 0, 0),
thickness=1)
s = e["contour"].copy()
s[:, :, 0] += coarse_pupil_width * 2
cv2.polylines(debug_img, [s],
isClosed=False,
color=(255, 255, 0),
thickness=1)
else:
if self._window:
s = e["contour"].copy()
s[:, :, 0] += coarse_pupil_width * 2
cv2.polylines(debug_img, [s],
isClosed=False,
color=(0, 0, 255),
thickness=1)
else:
if self._window:
cv2.polylines(debug_img, [s],
isClosed=False,
color=(0, 255, 255),
thickness=1)
# new_support = np.concatenate(new_support)
self.goodness.value = the_one['goodness']
###here we should AND original mask, selected contours with 2px thinkness (and 2px fitted ellipse -is the last one a good idea??)
support_mask = np.zeros(edges.shape, edges.dtype)
cv2.polylines(support_mask,
new_support,
isClosed=False,
color=(255, 255, 255),
thickness=2)
# #draw into the suport mast with thickness 2
new_edges = cv2.min(edges, support_mask)
new_contours = cv2.findNonZero(new_edges)
if self._window:
debug_img[0:support_mask.shape[0], 0:support_mask.shape[1],
2] = new_edges
###### do the ellipse fit and filter think again
ellipses = ((cv2.fitEllipse(c), c) for c in [new_contours])
ellipses = ((e, c) for e, c in ellipses
if (padding < e[0][1] < shape[0] -
padding and padding < e[0][0] < shape[1] - padding)
) # center is close to roi center
ellipses = ((e, c) for e, c in ellipses
if binary_img[e[0][1],
e[0][0]]) # center is on a dark pixel
ellipses = [(size_deviation(e, self.target_size.value), e, c)
for e, c in ellipses
if is_round(e, self.target_ratio)] # roundness test
for size_dif, e, c in ellipses:
pupil_ellipse = {}
pupil_ellipse['contour'] = c
a, b = e[1][0] / 2., e[1][
1] / 2. # majar minor radii of candidate ellipse
# pupil_ellipse['circumference'] = np.pi*abs(3*(a+b)-np.sqrt(10*a*b+3*(a**2+b**2)))
# pupil_ellipse['convex_hull'] = cv2.convexHull(pupil_ellipse['contour'])
pupil_ellipse['contour_area'] = cv2.contourArea(
cv2.convexHull(c))
pupil_ellipse['ellipse_area'] = np.pi * a * b
# print abs(pupil_ellipse['contour_area']-pupil_ellipse['ellipse_area'])
if abs(pupil_ellipse['contour_area'] -
pupil_ellipse['ellipse_area']) < 10:
pupil_ellipse[
'goodness'] = 0 #perfect match we'll take this one
else:
pupil_ellipse['goodness'] = size_dif
if visualize:
pass
# cv2.drawContours(pupil_img,[cv2.convexHull(c)],-1,(size_dif,size_dif,255))
# cv2.drawContours(pupil_img,[c],-1,(size_dif,size_dif,255))
pupil_ellipse['center'] = u_roi.add_vector(
p_roi.add_vector(e[0])) # compensate for roi offsets
pupil_ellipse['angle'] = e[-1]
pupil_ellipse['axes'] = e[1]
pupil_ellipse['major'] = max(e[1])
pupil_ellipse['minor'] = min(e[1])
pupil_ellipse[
'ratio'] = pupil_ellipse['minor'] / pupil_ellipse['major']
pupil_ellipse['norm_pupil'] = normalize(
pupil_ellipse['center'], (img.shape[1], img.shape[0]),
flip_y=True)
pupil_ellipse['timestamp'] = frame.timestamp
result = [
pupil_ellipse,
]
# the_new_one = result[0]
#done - if the new ellipse is good, we just overwrote the old result
if self._window:
self.gl_display_in_window(debug_img)
if self.should_sleep:
# sleep(3)
self.should_sleep = False
if result:
# update the target size
if result[0]['goodness'] >= 3: # perfect match!
self.target_size.value = result[0]['major']
else:
self.target_size.value = self.target_size.value + .2 * (
result[0]['major'] - self.target_size.value)
result.sort(
key=lambda e: abs(e['major'] - self.target_size.value))
if visualize:
pass
return result[0]
else:
self.goodness.value = 100
no_result = {}
no_result['timestamp'] = frame.timestamp
no_result['norm_pupil'] = None
return no_result
def eye(g_pool):
"""
this process needs a docstring
"""
# # callback functions
def on_resize(w, h):
atb.TwWindowSize(w, h)
adjust_gl_view(w, h)
def on_key(key, pressed):
if not atb.TwEventKeyboardGLFW(key, pressed):
if pressed:
if key == GLFW_KEY_ESC:
on_close()
def on_char(char, pressed):
if not atb.TwEventCharGLFW(char, pressed):
pass
def on_button(button, pressed):
if not atb.TwEventMouseButtonGLFW(button, pressed):
if bar.draw_roi.value:
if pressed:
pos = glfwGetMousePos()
pos = normalize(pos, glfwGetWindowSize())
pos = denormalize(pos, (img.shape[1], img.shape[0])) # pos in img pixels
r.setStart(pos)
bar.draw_roi.value = 1
else:
bar.draw_roi.value = 0
def on_pos(x, y):
if atb.TwMouseMotion(x, y):
pass
if bar.draw_roi.value == 1:
pos = glfwGetMousePos()
pos = normalize(pos, glfwGetWindowSize())
pos = denormalize(pos, (img.shape[1], img.shape[0])) # pos in img pixels
r.setEnd(pos)
def on_scroll(pos):
if not atb.TwMouseWheel(pos):
pass
def on_close():
g_pool.quit.value = True
print "EYE Process closing from window"
# initialize capture, check if it works
cap = autoCreateCapture(g_pool.eye_src, g_pool.eye_size)
if cap is None:
print "EYE: Error could not create Capture"
return
s, img = cap.read_RGB()
if not s:
print "EYE: Error could not get image"
return
height, width = img.shape[:2]
# pupil object
pupil = Temp()
pupil.norm_coords = (0.0, 0.0)
pupil.image_coords = (0.0, 0.0)
pupil.ellipse = None
pupil.gaze_coords = (0.0, 0.0)
try:
pupil.pt_cloud = np.load("cal_pt_cloud.npy")
map_pupil = get_map_from_cloud(pupil.pt_cloud, g_pool.world_size) ###world video size here
except:
pupil.pt_cloud = None
def map_pupil(vector):
return vector
r = Roi(img.shape)
p_r = Roi(img.shape)
# local object
l_pool = Temp()
l_pool.calib_running = False
l_pool.record_running = False
l_pool.record_positions = []
l_pool.record_path = None
l_pool.writer = None
l_pool.region_r = 20
atb.init()
bar = Bar(
"Eye",
g_pool,
dict(
label="Controls",
help="eye detection controls",
color=(50, 50, 50),
alpha=100,
text="light",
position=(10, 10),
refresh=0.1,
size=(200, 300),
),
)
# add 4vl2 camera controls to a seperate ATB bar
if cap.controls is not None:
c_bar = atb.Bar(
name="Camera_Controls",
label=cap.name,
help="UVC Camera Controls",
color=(50, 50, 50),
alpha=100,
text="light",
position=(220, 10),
refresh=2.0,
size=(200, 200),
)
# c_bar.add_var("auto_refresher",vtype=atb.TW_TYPE_BOOL8,getter=cap.uvc_refresh_all,setter=None,readonly=True)
# c_bar.define(definition='visible=0', varname="auto_refresher")
sorted_controls = [c for c in cap.controls.itervalues()]
sorted_controls.sort(key=lambda c: c.order)
for control in sorted_controls:
name = control.atb_name
if control.type == "bool":
c_bar.add_var(name, vtype=atb.TW_TYPE_BOOL8, getter=control.get_val, setter=control.set_val)
elif control.type == "int":
c_bar.add_var(name, vtype=atb.TW_TYPE_INT32, getter=control.get_val, setter=control.set_val)
c_bar.define(definition="min=" + str(control.min), varname=name)
c_bar.define(definition="max=" + str(control.max), varname=name)
c_bar.define(definition="step=" + str(control.step), varname=name)
elif control.type == "menu":
if control.menu is None:
vtype = None
else:
vtype = atb.enum(name, control.menu)
c_bar.add_var(name, vtype=vtype, getter=control.get_val, setter=control.set_val)
if control.menu is None:
c_bar.define(definition="min=" + str(control.min), varname=name)
c_bar.define(definition="max=" + str(control.max), varname=name)
c_bar.define(definition="step=" + str(control.step), varname=name)
else:
pass
if control.flags == "inactive":
pass
# c_bar.define(definition='readonly=1',varname=control.name)
c_bar.add_button("refresh", cap.update_from_device)
c_bar.add_button("load defaults", cap.load_defaults)
else:
c_bar = None
# Initialize glfw
glfwInit()
glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("Eye")
glfwSetWindowPos(800, 0)
if isinstance(g_pool.eye_src, str):
glfwSwapInterval(0) # turn of v-sync when using video as src for benchmarking
# register callbacks
glfwSetWindowSizeCallback(on_resize)
glfwSetWindowCloseCallback(on_close)
glfwSetKeyCallback(on_key)
glfwSetCharCallback(on_char)
glfwSetMouseButtonCallback(on_button)
glfwSetMousePosCallback(on_pos)
glfwSetMouseWheelCallback(on_scroll)
# gl_state settings
import OpenGL.GL as gl
gl.glEnable(gl.GL_POINT_SMOOTH)
gl.glPointSize(20)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
gl.glEnable(gl.GL_BLEND)
del gl
# event loop
while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value:
bar.update_fps()
s, img = cap.read_RGB()
sleep(bar.sleep.value) # for debugging only
###IMAGE PROCESSING
gray_img = grayscale(img[r.lY : r.uY, r.lX : r.uX])
integral = cv2.integral(gray_img)
integral = np.array(integral, dtype=c_float)
x, y, w = eye_filter(integral)
if w > 0:
p_r.set((y, x, y + w, x + w))
else:
p_r.set((0, 0, -1, -1))
# create view into the gray_img with the bounds of the rough pupil estimation
pupil_img = gray_img[p_r.lY : p_r.uY, p_r.lX : p_r.uX]
# pupil_img = cv2.morphologyEx(pupil_img, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT,(5,5)),iterations=2)
if True:
hist = cv2.calcHist(
[pupil_img], [0], None, [256], [0, 256]
) # (images, channels, mask, histSize, ranges[, hist[, accumulate]])
bins = np.arange(hist.shape[0])
spikes = bins[hist[:, 0] > 40] # every color seen in more than 40 pixels
if spikes.shape[0] > 0:
lowest_spike = spikes.min()
offset = 40
##display the histogram
sx, sy = 100, 1
colors = ((255, 0, 0), (0, 0, 255), (0, 255, 255))
h, w, chan = img.shape
# normalize
hist *= 1.0 / hist.max()
for i, h in zip(bins, hist[:, 0]):
c = colors[1]
cv2.line(img, (w, int(i * sy)), (w - int(h * sx), int(i * sy)), c)
cv2.line(img, (w, int(lowest_spike * sy)), (int(w - 0.5 * sx), int(lowest_spike * sy)), colors[0])
cv2.line(
img,
(w, int((lowest_spike + offset) * sy)),
(int(w - 0.5 * sx), int((lowest_spike + offset) * sy)),
colors[2],
)
# # k-means on the histogram finds peaks but thats no good for us...
# term_crit = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
# compactness, bestLabels, centers = cv2.kmeans(data=hist, K=2, criteria=term_crit, attempts=10, flags=cv2.KMEANS_RANDOM_CENTERS)
# cv2.line(img,(0,1),(int(compactness),1),(0,0,0))
# good_cluster = np.argmax(centers)
# # A = hist[bestLabels.ravel() == good_cluster]
# # B = hist[bestLabels.ravel() != good_cluster]
# bins = np.arange(hist.shape[0])
# good_bins = bins[bestLabels.ravel() == good_cluster]
# good_bins_mean = good_bins.sum()/good_bins.shape[0]
# good_bins_min = good_bins.min()
# h,w,chan = img.shape
# for h, i, label in zip(hist[:,0],range(hist.shape[0]), bestLabels.ravel()):
# c = colors[label]
# cv2.line(img,(w,int(i*sy)),(w-int(h*sx),int(i*sy)),c)
else:
# direct k-means on the image is best but expensive
Z = pupil_img[:: w / 30 + 1, :: w / 30 + 1].reshape((-1, 1))
Z = np.float32(Z)
# define criteria, number of clusters(K) and apply kmeans()
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 20, 2.0)
K = 5
ret, label, center = cv2.kmeans(Z, K, criteria, 10, cv2.KMEANS_RANDOM_CENTERS)
offset = 0
center.sort(axis=0)
lowest_spike = int(center[1])
# # Now convert back into uint8, and make original image
# center = np.uint8(center)
# res = center[label.flatten()]
# binary_img = res.reshape((pupil_img.shape))
# binary_img = bin_thresholding(binary_img,image_upper=res.min()+1)
# bar.bin_thresh.value = res.min()+1
bar.bin_thresh.value = lowest_spike
binary_img = bin_thresholding(pupil_img, image_upper=lowest_spike + offset)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))
cv2.dilate(binary_img, kernel, binary_img, iterations=2)
spec_mask = bin_thresholding(pupil_img, image_upper=250)
cv2.erode(spec_mask, kernel, spec_mask, iterations=1)
if bar.blur.value > 1:
pupil_img = cv2.medianBlur(pupil_img, bar.blur.value)
# create contours using Canny edge dectetion
contours = cv2.Canny(
pupil_img,
bar.canny_thresh.value,
bar.canny_thresh.value * bar.canny_ratio.value,
apertureSize=bar.canny_aperture.value,
)
# remove contours in areas not dark enough and where the glint (spectral refelction from IR leds)
contours = cv2.min(contours, spec_mask)
contours = cv2.min(contours, binary_img)
# Ellipse fitting from countours
result = fit_ellipse(
img[r.lY : r.uY, r.lX : r.uX][p_r.lY : p_r.uY, p_r.lX : p_r.uX],
contours,
binary_img,
target_size=bar.pupil_size.value,
size_tolerance=bar.pupil_size_tolerance.value,
)
# # Vizualizations
overlay = cv2.cvtColor(pupil_img, cv2.COLOR_GRAY2RGB) # create an RGB view onto the gray pupil ROI
overlay[:, :, 0] = cv2.max(pupil_img, contours) # green channel
overlay[:, :, 2] = cv2.max(pupil_img, binary_img) # blue channel
overlay[:, :, 1] = cv2.min(pupil_img, spec_mask) # red channel
# draw a blue dotted frame around the automatic pupil ROI in overlay...
overlay[::2, 0] = 0, 0, 255
overlay[::2, -1] = 0, 0, 255
overlay[0, ::2] = 0, 0, 255
overlay[-1, ::2] = 0, 0, 255
# and a solid (white) frame around the user defined ROI
gray_img[:, 0] = 255
gray_img[:, -1] = 255
gray_img[0, :] = 255
gray_img[-1, :] = 255
if bar.display.value == 0:
img = img
elif bar.display.value == 1:
img[r.lY : r.uY, r.lX : r.uX] = cv2.cvtColor(gray_img, cv2.COLOR_GRAY2RGB)
elif bar.display.value == 2:
img[r.lY : r.uY, r.lX : r.uX] = cv2.cvtColor(gray_img, cv2.COLOR_GRAY2RGB)
img[r.lY : r.uY, r.lX : r.uX][p_r.lY : p_r.uY, p_r.lX : p_r.uX] = overlay
elif bar.display.value == 3:
img = cv2.cvtColor(pupil_img, cv2.COLOR_GRAY2RGB)
else:
pass
if result is not None:
pupil.ellipse, others = result
pupil.image_coords = r.add_vector(p_r.add_vector(pupil.ellipse["center"]))
# update pupil size,angle and ratio for the ellipse filter algorithm
bar.pupil_size.value = bar.pupil_size.value + 0.5 * (pupil.ellipse["major"] - bar.pupil_size.value)
bar.pupil_ratio.value = bar.pupil_ratio.value + 0.7 * (pupil.ellipse["ratio"] - bar.pupil_ratio.value)
bar.pupil_angle.value = bar.pupil_angle.value + 1.0 * (pupil.ellipse["angle"] - bar.pupil_angle.value)
# if pupil found tighten the size tolerance
bar.pupil_size_tolerance.value -= 1
bar.pupil_size_tolerance.value = max(10, min(50, bar.pupil_size_tolerance.value))
# clamp pupil size
bar.pupil_size.value = max(20, min(300, bar.pupil_size.value))
# normalize
pupil.norm_coords = normalize(pupil.image_coords, (img.shape[1], img.shape[0]), flip_y=True)
# from pupil to gaze
pupil.gaze_coords = map_pupil(pupil.norm_coords)
g_pool.gaze_x.value, g_pool.gaze_y.value = pupil.gaze_coords
else:
pupil.ellipse = None
g_pool.gaze_x.value, g_pool.gaze_y.value = 0.0, 0.0
pupil.gaze_coords = None # whithout this line the last know pupil position is recorded if none is found
bar.pupil_size_tolerance.value += 1
###CALIBRATION###
# Initialize Calibration (setup variables and lists)
if g_pool.calibrate.value and not l_pool.calib_running:
l_pool.calib_running = True
pupil.pt_cloud = []
# While Calibrating...
if l_pool.calib_running and ((g_pool.ref_x.value != 0) or (g_pool.ref_y.value != 0)) and pupil.ellipse:
pupil.pt_cloud.append([pupil.norm_coords[0], pupil.norm_coords[1], g_pool.ref_x.value, g_pool.ref_y.value])
# Calculate mapping coefs
if not g_pool.calibrate.value and l_pool.calib_running:
l_pool.calib_running = 0
if pupil.pt_cloud: # some data was actually collected
print "Calibrating with", len(pupil.pt_cloud), "collected data points."
map_pupil = get_map_from_cloud(np.array(pupil.pt_cloud), g_pool.world_size, verbose=True)
np.save("cal_pt_cloud.npy", np.array(pupil.pt_cloud))
###RECORDING###
# Setup variables and lists for recording
if g_pool.pos_record.value and not l_pool.record_running:
l_pool.record_path = g_pool.eye_rx.recv()
print "l_pool.record_path: ", l_pool.record_path
video_path = path.join(l_pool.record_path, "eye.avi")
# FFV1 -- good speed lossless big file
# DIVX -- good speed good compression medium file
if bar.record_eye.value:
l_pool.writer = cv2.VideoWriter(
video_path, cv2.cv.CV_FOURCC(*"DIVX"), bar.fps.value, (img.shape[1], img.shape[0])
)
l_pool.record_positions = []
l_pool.record_running = True
# While recording...
if l_pool.record_running:
if pupil.gaze_coords is not None:
l_pool.record_positions.append(
[
pupil.gaze_coords[0],
pupil.gaze_coords[1],
pupil.norm_coords[0],
pupil.norm_coords[1],
bar.dt,
g_pool.frame_count_record.value,
]
)
if l_pool.writer is not None:
l_pool.writer.write(cv2.cvtColor(img, cv2.cv.COLOR_BGR2RGB))
# Done Recording: Save values and flip switch to off for recording
if not g_pool.pos_record.value and l_pool.record_running:
positions_path = path.join(l_pool.record_path, "gaze_positions.npy")
cal_pt_cloud_path = path.join(l_pool.record_path, "cal_pt_cloud.npy")
np.save(positions_path, np.asarray(l_pool.record_positions))
try:
np.save(cal_pt_cloud_path, np.asarray(pupil.pt_cloud))
except:
print "Warning: No calibration data associated with this recording."
l_pool.writer = None
l_pool.record_running = False
### GL-drawing
clear_gl_screen()
draw_gl_texture(img)
if bar.draw_pupil and pupil.ellipse:
pts = cv2.ellipse2Poly(
(int(pupil.image_coords[0]), int(pupil.image_coords[1])),
(int(pupil.ellipse["axes"][0] / 2), int(pupil.ellipse["axes"][1] / 2)),
int(pupil.ellipse["angle"]),
0,
360,
15,
)
draw_gl_polyline(pts, (1.0, 0, 0, 0.5))
draw_gl_point_norm(pupil.norm_coords, (1.0, 0.0, 0.0, 0.5))
atb.draw()
glfwSwapBuffers()
# end while running
print "EYE Process closed"
r.save()
bar.save()
atb.terminate()
glfwCloseWindow()
glfwTerminate()
def detect(self,frame,u_roi,visualize=False):
#get the user_roi
img = frame.img
r_img = img[u_roi.lY:u_roi.uY,u_roi.lX:u_roi.uX]
gray_img = grayscale(r_img)
# coarse pupil detection
integral = cv2.integral(gray_img)
integral = np.array(integral,dtype=c_float)
x,y,w,response = eye_filter(integral,100,400)
p_roi = Roi(gray_img.shape)
if w>0:
p_roi.set((y,x,y+w,x+w))
else:
p_roi.set((0,0,-1,-1))
coarse_pupil_center = x+w/2.,y+w/2.
coarse_pupil_width = w/2.
padding = coarse_pupil_width/4.
pupil_img = gray_img[p_roi.lY:p_roi.uY,p_roi.lX:p_roi.uX]
# binary thresholding of pupil dark areas
hist = cv2.calcHist([pupil_img],[0],None,[256],[0,256]) #(images, channels, mask, histSize, ranges[, hist[, accumulate]])
bins = np.arange(hist.shape[0])
spikes = bins[hist[:,0]>40] # every intensity seen in more than 40 pixels
if spikes.shape[0] >0:
lowest_spike = spikes.min()
highest_spike = spikes.max()
else:
lowest_spike = 200
highest_spike = 255
offset = self.intensity_range.value
spectral_offset = 5
if visualize:
# display the histogram
sx,sy = 100,1
colors = ((0,0,255),(255,0,0),(255,255,0),(255,255,255))
h,w,chan = img.shape
hist *= 1./hist.max() # normalize for display
for i,h in zip(bins,hist[:,0]):
c = colors[1]
cv2.line(img,(w,int(i*sy)),(w-int(h*sx),int(i*sy)),c)
cv2.line(img,(w,int(lowest_spike*sy)),(int(w-.5*sx),int(lowest_spike*sy)),colors[0])
cv2.line(img,(w,int((lowest_spike+offset)*sy)),(int(w-.5*sx),int((lowest_spike+offset)*sy)),colors[2])
cv2.line(img,(w,int((highest_spike)*sy)),(int(w-.5*sx),int((highest_spike)*sy)),colors[0])
cv2.line(img,(w,int((highest_spike- spectral_offset )*sy)),(int(w-.5*sx),int((highest_spike - spectral_offset)*sy)),colors[3])
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9,9))
#open operation to remove eye lashes
pupil_img = cv2.morphologyEx(pupil_img, cv2.MORPH_OPEN, kernel)
# PARAMS = {}
# blob_detector = cv2.SimpleBlobDetector(**PARAMS)
# kps = blob_detector.detect(pupil_img)
# blur = cv2.GaussianBlur(pupil_img,(5,5),0)
blur = pupil_img
# ret3,th3 = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
ret3,th3 = cv2.threshold(blur,lowest_spike+offset,255,cv2.THRESH_BINARY)
# ret3,th3 = cv2.threshold(th3,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
th3 = cv2.Laplacian(th3,cv2.CV_64F)
edges = cv2.Canny(pupil_img,
self.canny_thresh.value,
self.canny_thresh.value*self.canny_ratio.value,
apertureSize= self.canny_aperture.value)
r_img[p_roi.lY:p_roi.uY,p_roi.lX:p_roi.uX,1] = th3
r_img[p_roi.lY:p_roi.uY,p_roi.lX:p_roi.uX,2] = edges
# for kp in kps:
# print kp.pt
# cv2.circle(r_img[p_roi.lY:p_roi.uY,p_roi.lX:p_roi.uX],tuple(map(int,kp.pt,)),10,(255,255,255))
no_result = {}
no_result['timestamp'] = frame.timestamp
no_result['norm_pupil'] = None
return no_result
def detect(self,frame,u_roi,visualize=False):
if self.window_should_open:
self.open_window()
if self.window_should_close:
self.close_window()
#get the user_roi
img = frame.img
r_img = img[u_roi.lY:u_roi.uY,u_roi.lX:u_roi.uX]
gray_img = grayscale(r_img)
# coarse pupil detection
integral = cv2.integral(gray_img)
integral = np.array(integral,dtype=c_float)
x,y,w,response = eye_filter(integral,self.coarse_filter_min,self.coarse_filter_max)
p_roi = Roi(gray_img.shape)
if w>0:
p_roi.set((y,x,y+w,x+w))
else:
p_roi.set((0,0,-1,-1))
coarse_pupil_center = x+w/2.,y+w/2.
coarse_pupil_width = w/2.
padding = coarse_pupil_width/4.
pupil_img = gray_img[p_roi.lY:p_roi.uY,p_roi.lX:p_roi.uX]
# binary thresholding of pupil dark areas
hist = cv2.calcHist([pupil_img],[0],None,[256],[0,256]) #(images, channels, mask, histSize, ranges[, hist[, accumulate]])
bins = np.arange(hist.shape[0])
spikes = bins[hist[:,0]>40] # every intensity seen in more than 40 pixels
if spikes.shape[0] >0:
lowest_spike = spikes.min()
highest_spike = spikes.max()
else:
lowest_spike = 200
highest_spike = 255
offset = self.intensity_range.value
spectral_offset = 5
if visualize:
# display the histogram
sx,sy = 100,1
colors = ((0,0,255),(255,0,0),(255,255,0),(255,255,255))
h,w,chan = img.shape
hist *= 1./hist.max() # normalize for display
for i,h in zip(bins,hist[:,0]):
c = colors[1]
cv2.line(img,(w,int(i*sy)),(w-int(h*sx),int(i*sy)),c)
cv2.line(img,(w,int(lowest_spike*sy)),(int(w-.5*sx),int(lowest_spike*sy)),colors[0])
cv2.line(img,(w,int((lowest_spike+offset)*sy)),(int(w-.5*sx),int((lowest_spike+offset)*sy)),colors[2])
cv2.line(img,(w,int((highest_spike)*sy)),(int(w-.5*sx),int((highest_spike)*sy)),colors[0])
cv2.line(img,(w,int((highest_spike- spectral_offset )*sy)),(int(w-.5*sx),int((highest_spike - spectral_offset)*sy)),colors[3])
# create dark and spectral glint masks
self.bin_thresh.value = lowest_spike
binary_img = bin_thresholding(pupil_img,image_upper=lowest_spike + offset)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7,7))
cv2.dilate(binary_img, kernel,binary_img, iterations=2)
spec_mask = bin_thresholding(pupil_img, image_upper=highest_spike - spectral_offset)
cv2.erode(spec_mask, kernel,spec_mask, iterations=1)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9,9))
#open operation to remove eye lashes
pupil_img = cv2.morphologyEx(pupil_img, cv2.MORPH_OPEN, kernel)
if self.blur.value >1:
pupil_img = cv2.medianBlur(pupil_img,self.blur.value)
edges = cv2.Canny(pupil_img,
self.canny_thresh.value,
self.canny_thresh.value*self.canny_ratio.value,
apertureSize= self.canny_aperture.value)
# edges = cv2.adaptiveThreshold(pupil_img,255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, self.canny_aperture.value, 7)
# remove edges in areas not dark enough and where the glint is (spectral refelction from IR leds)
edges = cv2.min(edges, spec_mask)
edges = cv2.min(edges,binary_img)
if visualize:
overlay = img[u_roi.lY:u_roi.uY,u_roi.lX:u_roi.uX][p_roi.lY:p_roi.uY,p_roi.lX:p_roi.uX]
chn_img = grayscale(overlay)
overlay[:,:,2] = cv2.max(chn_img,edges) #b channel
overlay[:,:,0] = cv2.max(chn_img,binary_img) #g channel
overlay[:,:,1] = cv2.min(chn_img,spec_mask) #b channel
pupil_img = frame.img[u_roi.lY:u_roi.uY,u_roi.lX:u_roi.uX][p_roi.lY:p_roi.uY,p_roi.lX:p_roi.uX]
# draw a frame around the automatic pupil ROI in overlay...
pupil_img[::2,0] = 255,255,255
pupil_img[::2,-1]= 255,255,255
pupil_img[0,::2] = 255,255,255
pupil_img[-1,::2]= 255,255,255
pupil_img[::2,padding] = 255,255,255
pupil_img[::2,-padding]= 255,255,255
pupil_img[padding,::2] = 255,255,255
pupil_img[-padding,::2]= 255,255,255
frame.img[u_roi.lY:u_roi.uY,u_roi.lX:u_roi.uX][p_roi.lY:p_roi.uY,p_roi.lX:p_roi.uX] = pupil_img
# from edges to contours
contours, hierarchy = cv2.findContours(edges,
mode=cv2.RETR_LIST,
method=cv2.CHAIN_APPROX_NONE,offset=(0,0)) #TC89_KCOS
# contours is a list containing array([[[108, 290]],[[111, 290]]], dtype=int32) shape=(number of points,1,dimension(2) )
### first we want to filter out the bad stuff
# to short
good_contours = [c for c in contours if c.shape[0]>self.min_contour_size]
# now we learn things about each contour though looking at the curvature. For this we need to simplyfy the contour
arprox_contours = [cv2.approxPolyDP(c,epsilon=1.5,closed=False) for c in good_contours]
# cv2.drawContours(pupil_img,good_contours,-1,(255,255,0))
# cv2.drawContours(pupil_img,arprox_contours,-1,(0,0,255))
if self._window:
debug_img = np.zeros(img.shape,img.dtype)
x_shift = coarse_pupil_width*2 #just vor display
color = zip(range(0,250,30),range(0,255,30)[::-1],range(230,250))
split_contours = []
for c in arprox_contours:
curvature = GetAnglesPolyline(c)
# print curvature
# we split whenever there is a real kink (abs(curvature)<right angle) or a change in the genreal direction
kink_idx = find_kink_and_dir_change(curvature,100)
# kinks,k_index = convexity_defect(c,curvature)
# print "kink_idx", kink_idx
segs = split_at_corner_index(c,kink_idx)
# print len(segs)
# segs.sort(key=lambda e:-len(e))
for s in segs:
split_contours.append(s)
if self._window:
c = color.pop(0)
color.append(c)
# if s.shape[0] >=5:
# cv2.polylines(debug_img,[s],isClosed=False,color=c)
s = s.copy()
s[:,:,1] += coarse_pupil_width*2
cv2.polylines(debug_img,[s],isClosed=False,color=c)
s[:,:,0] += x_shift
x_shift += 5
cv2.polylines(debug_img,[s],isClosed=False,color=c)
# return {'timestamp':frame.timestamp,'norm_pupil':None}
#these segments may now be smaller, we need to get rid of those not long enough for ellipse fitting
good_contours = [c for c in split_contours if c.shape[0]>=5]
# cv2.polylines(img,good_contours,isClosed=False,color=(255,255,0))
shape = edges.shape
ellipses = ((cv2.fitEllipse(c),c) for c in good_contours)
ellipses = ((e,c) for e,c in ellipses if (padding < e[0][1] < shape[0]-padding and padding< e[0][0] < shape[1]-padding)) # center is close to roi center
ellipses = ((e,c) for e,c in ellipses if binary_img[e[0][1],e[0][0]]) # center is on a dark pixel
ellipses = [(e,c) for e,c in ellipses if is_round(e,self.target_ratio)] # roundness test
result = []
for e,c in ellipses:
size_dif = size_deviation(e,self.target_size.value)
pupil_ellipse = {}
pupil_ellipse['contour'] = c
a,b = e[1][0]/2.,e[1][1]/2. # majar minor radii of candidate ellipse
pupil_ellipse['circumference'] = np.pi*abs(3*(a+b)-np.sqrt(10*a*b+3*(a**2+b**2)))
# pupil_ellipse['convex_hull'] = cv2.convexHull(pupil_ellipse['contour'])
pupil_ellipse['contour_area'] = cv2.contourArea(cv2.convexHull(c))
pupil_ellipse['ellipse_area'] = np.pi*a*b
# print abs(pupil_ellipse['contour_area']-pupil_ellipse['ellipse_area'])
if abs(pupil_ellipse['contour_area']-pupil_ellipse['ellipse_area']) <10:
pupil_ellipse['goodness'] = abs(pupil_ellipse['contour_area']-pupil_ellipse['ellipse_area'])/10 #perfect match we'll take this one
else:
pupil_ellipse['goodness'] = size_dif
if visualize:
pass
# cv2.drawContours(pupil_img,[cv2.convexHull(c)],-1,(size_dif,size_dif,255))
# cv2.drawContours(pupil_img,[c],-1,(size_dif,size_dif,255))
pupil_ellipse['pupil_center'] = e[0] # compensate for roi offsets
pupil_ellipse['center'] = u_roi.add_vector(p_roi.add_vector(e[0])) # compensate for roi offsets
pupil_ellipse['angle'] = e[-1]
pupil_ellipse['axes'] = e[1]
pupil_ellipse['major'] = max(e[1])
pupil_ellipse['minor'] = min(e[1])
pupil_ellipse['ratio'] = pupil_ellipse['minor']/pupil_ellipse['major']
pupil_ellipse['norm_pupil'] = normalize(pupil_ellipse['center'], (img.shape[1], img.shape[0]),flip_y=True )
pupil_ellipse['timestamp'] = frame.timestamp
result.append(pupil_ellipse)
#### adding support
if result:
result.sort(key=lambda e: e['goodness'])
# for now we assume that this contour is part of the pupil
the_one = result[0]
# (center, size, angle) = cv2.fitEllipse(the_one['contour'])
# print "itself"
distances = dist_pts_ellipse(cv2.fitEllipse(the_one['contour']),the_one['contour'])
# print np.average(distances)
# print np.sum(distances)/float(distances.shape[0])
# print "other"
# if self._window:
# cv2.polylines(debug_img,[result[-1]['contour']],isClosed=False,color=(255,255,255),thickness=3)
with_another = np.concatenate((result[-1]['contour'],the_one['contour']))
distances = dist_pts_ellipse(cv2.fitEllipse(with_another),with_another)
# if 1.5 > np.sum(distances)/float(distances.shape[0]):
# if self._window:
# cv2.polylines(debug_img,[result[-1]['contour']],isClosed=False,color=(255,255,255),thickness=3)
perimeter_ratio = cv2.arcLength(the_one["contour"],closed=False)/the_one['circumference']
if perimeter_ratio > .9:
size_thresh = 0
eccentricity_thresh = 0
elif perimeter_ratio > .5:
size_thresh = the_one['major']/(5.)
eccentricity_thresh = the_one['major']/2.
self.should_sleep = True
else:
size_thresh = the_one['major']/(3.)
eccentricity_thresh = the_one['major']/2.
self.should_sleep = True
if self._window:
center = np.uint16(np.around(the_one['pupil_center']))
cv2.circle(debug_img,tuple(center),int(eccentricity_thresh),(0,255,0),1)
if self._window:
cv2.polylines(debug_img,[the_one["contour"]],isClosed=False,color=(255,0,0),thickness=2)
s = the_one["contour"].copy()
s[:,:,0] +=coarse_pupil_width*2
cv2.polylines(debug_img,[s],isClosed=False,color=(255,0,0),thickness=2)
# but are there other segments that could be used for support?
new_support = [the_one['contour'],]
if len(result)>1:
the_one = result[0]
target_axes = the_one['axes'][0]
# target_mean_curv = np.mean(curvature(the_one['contour'])
for e in result:
# with_another = np.concatenate((e['contour'],the_one['contour']))
# with_another = np.concatenate([r['contour'] for r in result])
with_another = e['contour']
distances = dist_pts_ellipse(cv2.fitEllipse(with_another),with_another)
# print np.std(distances)
thick = int(np.std(distances))
if 1.5 > np.average(distances) or 1:
if self._window:
# print thick
thick = min(20,thick)
cv2.polylines(debug_img,[e['contour']],isClosed=False,color=(255,255,255),thickness=thick)
if self._window:
cv2.polylines(debug_img,[e["contour"]],isClosed=False,color=(0,100,100))
center_dist = cv2.arcLength(np.array([the_one["pupil_center"],e['pupil_center']],dtype=np.int32),closed=False)
size_dif = abs(the_one['major']-e['major'])
# #lets make sure the countour is not behind the_one/'s coutour
# center_point = np.uint16(np.around(the_one['pupil_center']))
# other_center_point = np.uint16(np.around(e['pupil_center']))
# mid_point = the_one["contour"][the_one["contour"].shape[0]/2][0]
# other_mid_point = e["contour"][e["contour"].shape[0]/2][0]
# #reflect around mid_point
# p = center_point - mid_point
# p = np.array((-p[1],-p[0]))
# mir_center_point = p + mid_point
# dist_mid = cv2.arcLength(np.array([mid_point,other_mid_point]),closed=False)
# dist_center = cv2.arcLength(np.array([center_point,other_mid_point]),closed=False)
# if self._window:
# cv2.circle(debug_img,tuple(center_point),3,(0,255,0),2)
# cv2.circle(debug_img,tuple(other_center_point),2,(0,0,255),1)
# # cv2.circle(debug_img,tuple(mir_center_point),3,(0,255,0),2)
# # cv2.circle(debug_img,tuple(mid_point),2,(0,255,0),1)
# # cv2.circle(debug_img,tuple(other_mid_point),2,(0,0,255),1)
# cv2.polylines(debug_img,[np.array([center_point,other_mid_point]),np.array([mid_point,other_mid_point])],isClosed=False,color=(0,255,0))
if center_dist < eccentricity_thresh:
# print dist_mid-dist_center
# if dist_mid > dist_center-20:
if size_dif < size_thresh:
new_support.append(e["contour"])
if self._window:
cv2.polylines(debug_img,[s],isClosed=False,color=(255,0,0),thickness=1)
s = e["contour"].copy()
s[:,:,0] +=coarse_pupil_width*2
cv2.polylines(debug_img,[s],isClosed=False,color=(255,255,0),thickness=1)
else:
if self._window:
s = e["contour"].copy()
s[:,:,0] +=coarse_pupil_width*2
cv2.polylines(debug_img,[s],isClosed=False,color=(0,0,255),thickness=1)
else:
if self._window:
cv2.polylines(debug_img,[s],isClosed=False,color=(0,255,255),thickness=1)
# new_support = np.concatenate(new_support)
self.goodness.value = the_one['goodness']
###here we should AND original mask, selected contours with 2px thinkness (and 2px fitted ellipse -is the last one a good idea??)
support_mask = np.zeros(edges.shape,edges.dtype)
cv2.polylines(support_mask,new_support,isClosed=False,color=(255,255,255),thickness=2)
# #draw into the suport mast with thickness 2
new_edges = cv2.min(edges, support_mask)
new_contours = cv2.findNonZero(new_edges)
if self._window:
debug_img[0:support_mask.shape[0],0:support_mask.shape[1],2] = new_edges
###### do the ellipse fit and filter think again
ellipses = ((cv2.fitEllipse(c),c) for c in [new_contours])
ellipses = ((e,c) for e,c in ellipses if (padding < e[0][1] < shape[0]-padding and padding< e[0][0] < shape[1]-padding)) # center is close to roi center
ellipses = ((e,c) for e,c in ellipses if binary_img[e[0][1],e[0][0]]) # center is on a dark pixel
ellipses = [(size_deviation(e,self.target_size.value),e,c) for e,c in ellipses if is_round(e,self.target_ratio)] # roundness test
for size_dif,e,c in ellipses:
pupil_ellipse = {}
pupil_ellipse['contour'] = c
a,b = e[1][0]/2.,e[1][1]/2. # majar minor radii of candidate ellipse
# pupil_ellipse['circumference'] = np.pi*abs(3*(a+b)-np.sqrt(10*a*b+3*(a**2+b**2)))
# pupil_ellipse['convex_hull'] = cv2.convexHull(pupil_ellipse['contour'])
pupil_ellipse['contour_area'] = cv2.contourArea(cv2.convexHull(c))
pupil_ellipse['ellipse_area'] = np.pi*a*b
# print abs(pupil_ellipse['contour_area']-pupil_ellipse['ellipse_area'])
if abs(pupil_ellipse['contour_area']-pupil_ellipse['ellipse_area']) <10:
pupil_ellipse['goodness'] = 0 #perfect match we'll take this one
else:
pupil_ellipse['goodness'] = size_dif
if visualize:
pass
# cv2.drawContours(pupil_img,[cv2.convexHull(c)],-1,(size_dif,size_dif,255))
# cv2.drawContours(pupil_img,[c],-1,(size_dif,size_dif,255))
pupil_ellipse['center'] = u_roi.add_vector(p_roi.add_vector(e[0])) # compensate for roi offsets
pupil_ellipse['angle'] = e[-1]
pupil_ellipse['axes'] = e[1]
pupil_ellipse['major'] = max(e[1])
pupil_ellipse['minor'] = min(e[1])
pupil_ellipse['ratio'] = pupil_ellipse['minor']/pupil_ellipse['major']
pupil_ellipse['norm_pupil'] = normalize(pupil_ellipse['center'], (img.shape[1], img.shape[0]),flip_y=True )
pupil_ellipse['timestamp'] = frame.timestamp
result = [pupil_ellipse,]
# the_new_one = result[0]
#done - if the new ellipse is good, we just overwrote the old result
if self._window:
self.gl_display_in_window(debug_img)
if self.should_sleep:
# sleep(3)
self.should_sleep = False
if result:
# update the target size
if result[0]['goodness'] >=3: # perfect match!
self.target_size.value = result[0]['major']
else:
self.target_size.value = self.target_size.value + .2 * (result[0]['major']-self.target_size.value)
result.sort(key=lambda e: abs(e['major']-self.target_size.value))
if visualize:
pass
return result[0]
else:
self.goodness.value = 100
no_result = {}
no_result['timestamp'] = frame.timestamp
no_result['norm_pupil'] = None
return no_result
