def processFrame(img, facecascade, lefteyecascade, righteyecascade): """Process a single frame. """ gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) gray = cv2.equalizeHist(gray) facerects = detect(gray, facecascade) vis = img.copy() draw_rects(vis, facerects, (0, 255, 0)) facerect = None eyesubrectsleft, eyesubrectsright = [], [] eyesubrectleft, eyesubrectright = None, None leftEllipseBox, rightEllipseBox = None, None gazeLoc = None try: # find largest face rect and process maxsize = 0 maxidx = 0 for i in range(len(facerects)): a = rectArea(facerects[i]) if maxsize < a: maxsize = a maxidx = i facerect = facerects[maxidx] x1, y1, x2, y2 = facerect # left eye roileft = gray[y1:(y1+y2)/2, x1:(x1+x2)/2] vis_roileft = vis[y1:(y1+y2)/2, x1:(x1+x2)/2] gray_roileft = gray[y1:(y1+y2)/2, x1:(x1+x2)/2] eyesubrectsleft = detect(roileft.copy(), lefteyecascade) draw_rects(vis_roileft, eyesubrectsleft, (255, 0, 0)) # right eye roiright = gray[y1:(y1+y2)/2, (x1+x2)/2:x2] vis_roiright = vis[y1:(y1+y2)/2, (x1+x2)/2:x2] gray_roiright = gray[y1:(y1+y2)/2, (x1+x2)/2:x2] eyesubrectsright = detect(roiright.copy(), righteyecascade) draw_rects(vis_roiright, eyesubrectsright, (255, 0, 0)) # if left and right eyes detected if len(eyesubrectsleft) > 0 and len(eyesubrectsright) > 0: # take largest eye detected on each side maxsize = 0 maxidx = 0 for i in range(len(eyesubrectsleft)): a = rectArea(eyesubrectsleft[i]) if maxsize < a: maxsize = a maxidx = i eyesubrectleft = eyesubrectsleft[maxidx] maxsize = 0 maxidx = 0 for i in range(len(eyesubrectsright)): a = rectArea(eyesubrectsright[i]) if maxsize < a: maxsize = a maxidx = i eyesubrectright = eyesubrectsright[maxidx] # slice from color img vis_roileft_eye = vis_roileft[eyesubrectleft[1]:eyesubrectleft[3], eyesubrectleft[0]:eyesubrectleft[2]] vis_roiright_eye = vis_roiright[eyesubrectright[1]:eyesubrectright[3], eyesubrectright[0]:eyesubrectright[2]] # slice from gray img gray_roileft_eye = gray_roileft[eyesubrectleft[1]:eyesubrectleft[3], eyesubrectleft[0]:eyesubrectleft[2]] gray_roiright_eye = gray_roiright[eyesubrectright[1]:eyesubrectright[3], eyesubrectright[0]:eyesubrectright[2]] # display combined gray eyes (pre-processed) comb = combineEyes( gray_roileft_eye, gray_roiright_eye ) cv2.imshow('combinedeyes', comb) leftpupil = processEyeByCorners(eyesubrectleft, vis_roileft_eye, gray_roileft_eye, vis, gray) rightpupil = processEyeByCorners(eyesubrectright, vis_roiright_eye, gray_roiright_eye, vis, gray) if leftpupil != None and rightpupil != None: leftyaw, leftpitch, rightyaw, rightpitch = gazeFunctions.getAnglesFromPupilRelativeCenter(leftpupil, rightpupil, gazeFunctions.dummyCalibrationPoints, fliplr=True) gazeloc = gazeFunctions.findGazeLocation(leftyaw, leftpitch, rightyaw, rightpitch) print gazeloc #leftthresh, leftEllipseBox = processEye(eyesubrectleft, vis_roileft_eye, gray_roileft_eye, vis, gray) #rightthresh, rightEllipseBox = processEye(eyesubrectright, vis_roiright_eye, gray_roiright_eye, vis, gray) # #combthresh = combineEyes(leftthresh, rightthresh) #cv2.imshow('combinedthresh', combthresh) ############ Thresholding ########### # ## threshold left eye #leftthresh = gray_roileft_eye.copy() ##cv2.adaptiveThreshold(leftthresh, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 25, 25, leftthresh) #leftthresh = thresholdByPercentage(leftthresh, .075) # ## threshold right eye #rightthresh = gray_roiright_eye.copy() ##cv2.adaptiveThreshold(rightthresh, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 25, 25, rightthresh) #rightthresh = thresholdByPercentage(rightthresh, .075) # ## display combined threshes #combthresh = combineEyes(leftthresh, rightthresh) #cv2.imshow('combinedthresh', combthresh) # # ############ Contours ########### # ## find contours from thresholded img #contours, heirarchy = cv2.findContours(leftthresh.copy(),cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) # #maxarea = 0 #maxidx = 0 # ## find contour with largest area #for idx in range(len(contours)): # a = cv2.contourArea(contours[idx]) # if maxarea < a: # maxarea = a # maxidx = idx # #cv2.drawContours(vis_roileft_eye, contours, maxidx, (0,0,255), -1) # ##for idx in range(len(contours)): ## ## clr = (randrange(0, 255), randrange(0, 255), randrange(0, 255)) ## cv2.drawContours(vis_roileft_eye, contours, idx, clr, -1) # # ############ Ellipse ########### # ## fit ellipse to the detected contour #ellipseBox = cv2.fitEllipse(contours[maxidx]) #cv2.ellipse(vis_roileft_eye, ellipseBox, (50, 150, 255)) # ##pdb.set_trace() # ##combvis = combineEyes( ## vis_roileft_eye, ## vis_roiright_eye ## ) ##cv2.imshow('colorcombinedeyes', combvis) # ##pdb.set_trace() except IndexError: pass cv2.imshow('facedetect', vis) #return vis, gray, facerect, eyesubrectleft, eyesubrectright, leftEllipseBox, rightEllipseBox return vis, gray, facerect, eyesubrectleft, eyesubrectright, gazeLoc
def processFrame(img, facecascade, lefteyecascade, righteyecascade): """Process a single frame. """ gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) gray = cv2.equalizeHist(gray) facerects = detect(gray, facecascade) vis = img.copy() draw_rects(vis, facerects, (0, 255, 0)) facerect = None eyesubrectsleft, eyesubrectsright = [], [] eyesubrectleft, eyesubrectright = None, None leftEllipseBox, rightEllipseBox = None, None gazeLoc = None try: # find largest face rect and process maxsize = 0 maxidx = 0 for i in range(len(facerects)): a = rectArea(facerects[i]) if maxsize < a: maxsize = a maxidx = i facerect = facerects[maxidx] x1, y1, x2, y2 = facerect # left eye roileft = gray[y1:(y1 + y2) / 2, x1:(x1 + x2) / 2] vis_roileft = vis[y1:(y1 + y2) / 2, x1:(x1 + x2) / 2] gray_roileft = gray[y1:(y1 + y2) / 2, x1:(x1 + x2) / 2] eyesubrectsleft = detect(roileft.copy(), lefteyecascade) draw_rects(vis_roileft, eyesubrectsleft, (255, 0, 0)) # right eye roiright = gray[y1:(y1 + y2) / 2, (x1 + x2) / 2:x2] vis_roiright = vis[y1:(y1 + y2) / 2, (x1 + x2) / 2:x2] gray_roiright = gray[y1:(y1 + y2) / 2, (x1 + x2) / 2:x2] eyesubrectsright = detect(roiright.copy(), righteyecascade) draw_rects(vis_roiright, eyesubrectsright, (255, 0, 0)) # if left and right eyes detected if len(eyesubrectsleft) > 0 and len(eyesubrectsright) > 0: # take largest eye detected on each side maxsize = 0 maxidx = 0 for i in range(len(eyesubrectsleft)): a = rectArea(eyesubrectsleft[i]) if maxsize < a: maxsize = a maxidx = i eyesubrectleft = eyesubrectsleft[maxidx] maxsize = 0 maxidx = 0 for i in range(len(eyesubrectsright)): a = rectArea(eyesubrectsright[i]) if maxsize < a: maxsize = a maxidx = i eyesubrectright = eyesubrectsright[maxidx] # slice from color img vis_roileft_eye = vis_roileft[eyesubrectleft[1]:eyesubrectleft[3], eyesubrectleft[0]:eyesubrectleft[2]] vis_roiright_eye = vis_roiright[ eyesubrectright[1]:eyesubrectright[3], eyesubrectright[0]:eyesubrectright[2]] # slice from gray img gray_roileft_eye = gray_roileft[ eyesubrectleft[1]:eyesubrectleft[3], eyesubrectleft[0]:eyesubrectleft[2]] gray_roiright_eye = gray_roiright[ eyesubrectright[1]:eyesubrectright[3], eyesubrectright[0]:eyesubrectright[2]] # display combined gray eyes (pre-processed) comb = combineEyes(gray_roileft_eye, gray_roiright_eye) cv2.imshow('combinedeyes', comb) leftpupil = processEyeByCorners(eyesubrectleft, vis_roileft_eye, gray_roileft_eye, vis, gray) rightpupil = processEyeByCorners(eyesubrectright, vis_roiright_eye, gray_roiright_eye, vis, gray) if leftpupil != None and rightpupil != None: leftyaw, leftpitch, rightyaw, rightpitch = gazeFunctions.getAnglesFromPupilRelativeCenter( leftpupil, rightpupil, gazeFunctions.dummyCalibrationPoints, fliplr=True) gazeloc = gazeFunctions.findGazeLocation( leftyaw, leftpitch, rightyaw, rightpitch) print gazeloc #leftthresh, leftEllipseBox = processEye(eyesubrectleft, vis_roileft_eye, gray_roileft_eye, vis, gray) #rightthresh, rightEllipseBox = processEye(eyesubrectright, vis_roiright_eye, gray_roiright_eye, vis, gray) # #combthresh = combineEyes(leftthresh, rightthresh) #cv2.imshow('combinedthresh', combthresh) ############ Thresholding ########### # ## threshold left eye #leftthresh = gray_roileft_eye.copy() ##cv2.adaptiveThreshold(leftthresh, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 25, 25, leftthresh) #leftthresh = thresholdByPercentage(leftthresh, .075) # ## threshold right eye #rightthresh = gray_roiright_eye.copy() ##cv2.adaptiveThreshold(rightthresh, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 25, 25, rightthresh) #rightthresh = thresholdByPercentage(rightthresh, .075) # ## display combined threshes #combthresh = combineEyes(leftthresh, rightthresh) #cv2.imshow('combinedthresh', combthresh) # # ############ Contours ########### # ## find contours from thresholded img #contours, heirarchy = cv2.findContours(leftthresh.copy(),cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) # #maxarea = 0 #maxidx = 0 # ## find contour with largest area #for idx in range(len(contours)): # a = cv2.contourArea(contours[idx]) # if maxarea < a: # maxarea = a # maxidx = idx # #cv2.drawContours(vis_roileft_eye, contours, maxidx, (0,0,255), -1) # ##for idx in range(len(contours)): ## ## clr = (randrange(0, 255), randrange(0, 255), randrange(0, 255)) ## cv2.drawContours(vis_roileft_eye, contours, idx, clr, -1) # # ############ Ellipse ########### # ## fit ellipse to the detected contour #ellipseBox = cv2.fitEllipse(contours[maxidx]) #cv2.ellipse(vis_roileft_eye, ellipseBox, (50, 150, 255)) # ##pdb.set_trace() # ##combvis = combineEyes( ## vis_roileft_eye, ## vis_roiright_eye ## ) ##cv2.imshow('colorcombinedeyes', combvis) # ##pdb.set_trace() except IndexError: pass cv2.imshow('facedetect', vis) #return vis, gray, facerect, eyesubrectleft, eyesubrectright, leftEllipseBox, rightEllipseBox return vis, gray, facerect, eyesubrectleft, eyesubrectright, gazeLoc
def processFrame(self, img, simulatormode): """Process a single frame. """ self.simulatormode = simulatormode facerect = None eyesubrectsleft, eyesubrectsright = [], [] eyesubrectleft, eyesubrectright = None, None leftEllipseBox, rightEllipseBox = None, None gazeLoc = None # get gray version of img gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) gray = cv2.equalizeHist(gray) try: # detect face facerects = self._detect(gray, self.facecascade) vis = img.copy() if self.draw: self._drawRects(vis, facerects, (0, 255, 0)) # find largest face rect facerect = self._getLargestRect(facerects) x1, y1, x2, y2 = facerect # left eye roileft = gray[y1:(y1 + y2) / 2, x1:(x1 + x2) / 2] vis_roileft = vis[y1:(y1 + y2) / 2, x1:(x1 + x2) / 2] gray_roileft = gray[y1:(y1 + y2) / 2, x1:(x1 + x2) / 2] eyesubrectsleft = self._detect(roileft.copy(), self.lefteyecascade) if self.draw: self._drawRects(vis_roileft, eyesubrectsleft, (255, 0, 0)) # right eye roiright = gray[y1:(y1 + y2) / 2, (x1 + x2) / 2:x2] vis_roiright = vis[y1:(y1 + y2) / 2, (x1 + x2) / 2:x2] gray_roiright = gray[y1:(y1 + y2) / 2, (x1 + x2) / 2:x2] eyesubrectsright = self._detect(roiright.copy(), self.righteyecascade) if self.draw: self._drawRects(vis_roiright, eyesubrectsright, (255, 0, 0)) # take largest eye detected on each side eyesubrectleft = self._getLargestRect(eyesubrectsleft) eyesubrectright = self._getLargestRect(eyesubrectsright) # slice from color img vis_roileft_eye = vis_roileft[eyesubrectleft[1]:eyesubrectleft[3], eyesubrectleft[0]:eyesubrectleft[2]] vis_roiright_eye = vis_roiright[ eyesubrectright[1]:eyesubrectright[3], eyesubrectright[0]:eyesubrectright[2]] # slice from gray img gray_roileft_eye = gray_roileft[ eyesubrectleft[1]:eyesubrectleft[3], eyesubrectleft[0]:eyesubrectleft[2]] gray_roiright_eye = gray_roiright[ eyesubrectright[1]:eyesubrectright[3], eyesubrectright[0]:eyesubrectright[2]] if self.draw: # display combined gray eyes (pre-processed) comb = self._combineEyes(gray_roileft_eye, gray_roiright_eye) cv2.imshow('combinedeyes', comb) leftpupil = self._processEyeByCorners(eyesubrectleft, vis_roileft_eye, gray_roileft_eye, vis, gray) rightpupil = self._processEyeByCorners(eyesubrectright, vis_roiright_eye, gray_roiright_eye, vis, gray) if leftpupil != None and rightpupil != None: #pdb.set_trace() # if in CALIBRATION mode if self.simulatormode == 'CALIBRATION': self.calib.processPhase(leftpupil, rightpupil) else: leftyaw, leftpitch, rightyaw, rightpitch = gazeFunctions.getAnglesFromPupilRelativeCenter( leftpupil, rightpupil, self.calib.calibrationInfo, self.calib.calibrationPoints, fliplr=True) gazeLoc = gazeFunctions.findGazeLocation( leftyaw, leftpitch, rightyaw, rightpitch) if gazeLoc != None: self.lastGazeLoc = gazeLoc print gazeLoc else: print 'Gazing offscreen' except IndexError: pass if self.draw: cv2.imshow('facedetect', vis) return vis, gray, facerect, eyesubrectleft, eyesubrectright, gazeLoc
def processFrame(self, img, simulatormode): """Process a single frame. """ self.simulatormode = simulatormode facerect = None eyesubrectsleft, eyesubrectsright = [], [] eyesubrectleft, eyesubrectright = None, None leftEllipseBox, rightEllipseBox = None, None gazeLoc = None # get gray version of img gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) gray = cv2.equalizeHist(gray) try: # detect face facerects = self._detect(gray, self.facecascade) vis = img.copy() if self.draw: self._drawRects(vis, facerects, (0, 255, 0)) # find largest face rect facerect = self._getLargestRect(facerects) x1, y1, x2, y2 = facerect # left eye roileft = gray[y1:(y1+y2)/2, x1:(x1+x2)/2] vis_roileft = vis[y1:(y1+y2)/2, x1:(x1+x2)/2] gray_roileft = gray[y1:(y1+y2)/2, x1:(x1+x2)/2] eyesubrectsleft = self._detect(roileft.copy(), self.lefteyecascade) if self.draw: self._drawRects(vis_roileft, eyesubrectsleft, (255, 0, 0)) # right eye roiright = gray[y1:(y1+y2)/2, (x1+x2)/2:x2] vis_roiright = vis[y1:(y1+y2)/2, (x1+x2)/2:x2] gray_roiright = gray[y1:(y1+y2)/2, (x1+x2)/2:x2] eyesubrectsright = self._detect(roiright.copy(), self.righteyecascade) if self.draw: self._drawRects(vis_roiright, eyesubrectsright, (255, 0, 0)) # take largest eye detected on each side eyesubrectleft = self._getLargestRect(eyesubrectsleft) eyesubrectright = self._getLargestRect(eyesubrectsright) # slice from color img vis_roileft_eye = vis_roileft[eyesubrectleft[1]:eyesubrectleft[3], eyesubrectleft[0]:eyesubrectleft[2]] vis_roiright_eye = vis_roiright[eyesubrectright[1]:eyesubrectright[3], eyesubrectright[0]:eyesubrectright[2]] # slice from gray img gray_roileft_eye = gray_roileft[eyesubrectleft[1]:eyesubrectleft[3], eyesubrectleft[0]:eyesubrectleft[2]] gray_roiright_eye = gray_roiright[eyesubrectright[1]:eyesubrectright[3], eyesubrectright[0]:eyesubrectright[2]] if self.draw: # display combined gray eyes (pre-processed) comb = self._combineEyes( gray_roileft_eye, gray_roiright_eye ) cv2.imshow('combinedeyes', comb) leftpupil = self._processEyeByCorners(eyesubrectleft, vis_roileft_eye, gray_roileft_eye, vis, gray) rightpupil = self._processEyeByCorners(eyesubrectright, vis_roiright_eye, gray_roiright_eye, vis, gray) if leftpupil != None and rightpupil != None: #pdb.set_trace() # if in CALIBRATION mode if self.simulatormode == 'CALIBRATION': self.calib.processPhase(leftpupil, rightpupil) else: leftyaw, leftpitch, rightyaw, rightpitch = gazeFunctions.getAnglesFromPupilRelativeCenter(leftpupil, rightpupil, self.calib.calibrationInfo, self.calib.calibrationPoints, fliplr=True) gazeLoc = gazeFunctions.findGazeLocation(leftyaw, leftpitch, rightyaw, rightpitch) if gazeLoc != None: self.lastGazeLoc = gazeLoc print gazeLoc else: print 'Gazing offscreen' except IndexError: pass if self.draw: cv2.imshow('facedetect', vis) return vis, gray, facerect, eyesubrectleft, eyesubrectright, gazeLoc