def __init__ (self, enable_aws, enable_draw, e_cam_id, f_cam_id) :
self.enable_aws = enable_aws
self.enable_draw = enable_draw
self.frame_ct = 0
self.screencapid = 0
self.eyecam = cv.CaptureFromCAM(e_cam_id)
self.forwardcam = cv.CaptureFromCAM(f_cam_id)
self.font = cv.InitFont(cv.CV_FONT_HERSHEY_PLAIN, 1, 1, 0, 1, 8)
self.starttime = time.time()
self.previoustime = 0
self.threshold_toggle = False
self.threshold = DEFAULT_THRESHOLD
self.filterlevel = 0
self.queue = OCR_Queue()
if self.enable_draw :
self.eyecam_post = 'Eyecam Robovision'
cv.NamedWindow(self.eyecam_post, cv.CV_WINDOW_AUTOSIZE)
self.eyecam_raw = 'Eyecam Raw'
cv.NamedWindow(self.eyecam_raw, cv.CV_WINDOW_AUTOSIZE)
self.forward_window = 'Forward Vision'
cv.NamedWindow(self.forward_window, cv.CV_WINDOW_AUTOSIZE)
# Build an inital bounding box for the eye
sample_frame = cv.QueryFrame(self.eyecam)
self.frame_size = cv.GetSize(sample_frame)
self.build_default_bb()
self.line_tracker = Line_Detector()
self.last_frame_sent = None
if self.enable_draw :
cv.MoveWindow(self.eyecam_post, 0, 0)
cv.MoveWindow(self.eyecam_raw, sample_frame.width + 32, 0)
cv.MoveWindow(self.forward_window, sample_frame.width + 32, sample_frame.height + 32)
class Iris :
def __init__ (self, enable_aws, enable_draw, e_cam_id, f_cam_id) :
self.enable_aws = enable_aws
self.enable_draw = enable_draw
self.frame_ct = 0
self.screencapid = 0
self.eyecam = cv.CaptureFromCAM(e_cam_id)
self.forwardcam = cv.CaptureFromCAM(f_cam_id)
self.font = cv.InitFont(cv.CV_FONT_HERSHEY_PLAIN, 1, 1, 0, 1, 8)
self.starttime = time.time()
self.previoustime = 0
self.threshold_toggle = False
self.threshold = DEFAULT_THRESHOLD
self.filterlevel = 0
self.queue = OCR_Queue()
if self.enable_draw :
self.eyecam_post = 'Eyecam Robovision'
cv.NamedWindow(self.eyecam_post, cv.CV_WINDOW_AUTOSIZE)
self.eyecam_raw = 'Eyecam Raw'
cv.NamedWindow(self.eyecam_raw, cv.CV_WINDOW_AUTOSIZE)
self.forward_window = 'Forward Vision'
cv.NamedWindow(self.forward_window, cv.CV_WINDOW_AUTOSIZE)
# Build an inital bounding box for the eye
sample_frame = cv.QueryFrame(self.eyecam)
self.frame_size = cv.GetSize(sample_frame)
self.build_default_bb()
self.line_tracker = Line_Detector()
self.last_frame_sent = None
if self.enable_draw :
cv.MoveWindow(self.eyecam_post, 0, 0)
cv.MoveWindow(self.eyecam_raw, sample_frame.width + 32, 0)
cv.MoveWindow(self.forward_window, sample_frame.width + 32, sample_frame.height + 32)
def handle_keys (self) :
c = cv.WaitKey(2)
char = chr(c & 0xff)
if char == 'h' or char == 'H' or char =='?' :
print 'Iris key bindings:'
print 'p\tSave current image'
print 'c\tFake a line event (post a frame to AWS)'
print 'e\tEvaporate line event cooldown'
print 's\tSwap camera feeds'
print 't\tSet/unset threshold'
print 'f\tCycle filters'
print 'h\tDisplay this help'
print 'q\tQuit'
elif char == 'p' or char == 'P' :
cv.SaveImage('pupil_%02d.png' %(self.screencapid), self.pupil_frame)
self.screencapid += 1
elif char == 'f' or char == 'F' :
self.filterlevel += 1
self.filterlevel %= NUM_FILTERS
elif char == 't' :
self.threshold_toggle = not self.threshold_toggle
if self.threshold_toggle :
print 'Use the up and down keys to change the threshold'
else :
print 'Resetting threshold to', DEFAULT_THRESHOLD
self.threshold = DEFAULT_THRESHOLD
elif self.threshold_toggle and c == 0x10ff52 :
# Up arrow key
self.threshold += 5
self.threshold = min(self.threshold, 255)
print 'Threshold is', self.threshold
elif self.threshold_toggle and c == 0x10ff54 :
# Down arrow key
self.threshold -= 5
self.threshold = max(self.threshold, 0)
print 'Threshold is', self.threshold
elif char == 'c' or char == 'C' :
print 'Faking a line event'
to_send = cv.QueryFrame(self.forwardcam)
self.last_frame_sent = to_send
if self.enable_aws :
aws_thread = threading.Thread(target=send_to_aws,
args=(self.queue, to_send, time.time() - self.starttime))
aws_thread.daemon = True
aws_thread.start()
elif char == 's' or char == 'S' :
print 'Switching camera feeds'
self.line_tracker.reset_timer()
temp = self.eyecam
self.eyecam = self.forwardcam
self.forwardcam = temp
elif char == 'e' or char == 'E' :
print 'Evaporating cooldown'
self.line_tracker.evaporate_cooldown()
elif char == 'q' or char == 'Q' or c == 0x10001b :
print 'Thank you for using Iris'
return False
return True
def build_default_bb(self) :
upperleft = (self.frame_size[0]/4, self.frame_size[1]/4)
lowerright = (3*self.frame_size[0]/4, 3*self.frame_size[1]/4)
self.bounding_box = (upperleft, lowerright)
# pip is the picture in picture, the thing around which we're building a border
def build_border (self, pip) :
top_border = 0
left_border = 0
right_border = 0
bottom_border = 64
border_size = (pip.height + top_border + bottom_border,
pip.width + left_border + right_border, 3)
border = np.zeros(border_size, np.uint8) # Outer is a numpy array
pip_loc = (top_border, left_border)
blit(border, cv.GetMat(pip), pip_loc)
return cv.fromarray(border)
def handle_dark_pixels (self, grey_frame) :
# TODO: optimize?
# Track black pixels within the bounding box, and compute the average
dark_pix = []
avg_x = 0.0
avg_y = 0.0
for y in range(grey_frame.height):
for x in range(grey_frame.width):
if grey_frame[y, x] < 100 :
dark_pix.append((x, y))
avg_x += x
avg_y += y
if len(dark_pix) == 0 :
return (42, 42), 0
avg_x = int(avg_x / len(dark_pix))
avg_y = int(avg_y / len(dark_pix))
center = (avg_x, avg_y)
distances = []
dist_pix = []
for pix in dark_pix :
dist = math.sqrt((center[0] - pix[0]) ** 2 + (center[1] - center[1]) ** 2)
distances.append(dist)
dist_pix.append((dist, pix))
distances = np.array(distances)
dist_std = np.std(distances)
dist_avg = np.mean(distances)
trimmed_pix = []
avg_x = 0.0
avg_y = 0.0
for pix in dist_pix :
if pix[0] < dist_avg + 1 * dist_std :
trimmed_pix.append(pix[1])
avg_x += pix[1][0]
avg_y += pix[1][1]
if len(trimmed_pix) == 0 :
return (42, 42), 0
avg_x = int(avg_x / len(trimmed_pix))
avg_y = int(avg_y / len(trimmed_pix))
trimmed_center = (avg_x, avg_y)
return trimmed_center, int(dist_avg)
def find_pupil (self, frame) :
currenttime = time.time() - self.starttime
# Preprocess : greyscale, smooth, equalize, threshold
grey_frame = cv.CreateImage(cv.GetSize(frame), 8, 1)
g_plane = cv.CreateImage(cv.GetSize(frame), 8, 1)
b_plane = cv.CreateImage(cv.GetSize(frame), 8, 1)
# TODO: Just grab the blue portion, instead of greyscale
cv.Split(frame, None, g_plane, b_plane, None)
cv.AddWeighted(g_plane, .5, b_plane, .5, 0.0, grey_frame)
#cv.CvtColor(frame, grey_frame, cv.CV_BGR2GRAY)
if self.filterlevel == 1 :
out_frame = cv.CloneImage(g_plane)
label = 'Green Filter'
elif self.filterlevel == 2 :
out_frame = cv.CloneImage(b_plane)
label = 'Blue Filter'
elif self.filterlevel == 3 :
out_frame = cv.CloneImage(grey_frame)
label = 'Blue and Green'
cv.Smooth(grey_frame, grey_frame, cv.CV_MEDIAN)
cv.EqualizeHist(grey_frame, grey_frame)
if self.filterlevel == 4 :
out_frame = cv.CloneImage(grey_frame)
label = 'Smooth and Equalize'
color = 255
cv.Threshold(grey_frame, grey_frame, self.threshold,
color, cv.CV_THRESH_BINARY)
if self.filterlevel == 5 :
out_frame = cv.CloneImage(grey_frame)
label = 'Threshold'
# Dilate to remove eyebrows
element_shape = cv.CV_SHAPE_RECT
pos = 1
element = cv.CreateStructuringElementEx(pos*2+1, pos*2+1,
pos, pos, element_shape)
cv.Dilate(grey_frame, grey_frame, element, 2)
cv.Smooth(grey_frame, grey_frame, cv.CV_MEDIAN)
if self.filterlevel == 6 :
out_frame = cv.CloneImage(grey_frame)
label = 'Dilate'
# Track black pixels within the bounding box, and compute the average
center, radius = self.handle_dark_pixels(grey_frame)
if self.filterlevel == 0 :
out_frame = cv.CloneImage(grey_frame)
label = 'Full Robovision'
# See if we've found a line-event
if currenttime - self.previoustime > .05 :
event_detected = self.line_tracker.add_point(center)
if event_detected :
print 'Detected line event at %0.2f' %currenttime
to_send = cv.QueryFrame(self.forwardcam)
self.last_frame_sent = to_send
if self.enable_aws :
aws_thread = threading.Thread(target=send_to_aws,
args=(self.queue, to_send, currenttime))
aws_thread.daemon = True
aws_thread.start()
# Draw robovision elements on the frame
if self.enable_draw :
# Switch back to color encoding
pupil_frame = cv.CreateImage((frame.width, frame.height), 8, 3)
cv.CvtColor(out_frame, pupil_frame, cv.CV_GRAY2BGR)
if self.filterlevel == 0 :
# Mark the sequence of points that triggered the last line event
for point in self.line_tracker.get_last_sequence() :
cv.Circle(pupil_frame, point, 3, cv.CV_RGB(255, 0, 255))
# Dot the center of the pupil
cv.Circle(pupil_frame, center, 4, cv.CV_RGB(255, 0, 0), cv.CV_FILLED)
# Circle the full pupil
cv.Circle(pupil_frame, center, radius, cv.CV_RGB(255, 128, 0));
# Give it a border
pupil_border = self.build_border(pupil_frame)
cv.PutText(pupil_border,
'Frame %s - %0.2f seconds - %s - threshold %s' %(self.frame_ct, currenttime, label, self.threshold),
(10, frame.height+50), self.font, 255)
return center, pupil_border
else : # No display
return center, None
def repeat(self) :
currenttime = time.time() - self.starttime
box_w = self.bounding_box[1][0]-self.bounding_box[0][0]
box_h = self.bounding_box[1][1]-self.bounding_box[0][1]
frame = cv.GetSubRect(cv.QueryFrame(self.eyecam), (self.bounding_box[0][0], self.bounding_box[0][1], box_w, box_h)) # frame is an iplimage
self.frame_ct += 1
# Find the pupil
center, pupil_frame = self.find_pupil(frame)
# Draw the visualization, if needed
if self.enable_draw :
# Draw the eyecam feed with robovision
cv.ShowImage(self.eyecam_post, pupil_frame)
self.pupil_frame = pupil_frame
# Draw the eyecam feed w/o robovision
cv.ShowImage(self.eyecam_raw, frame)
# Draw the forwardcam feed
if self.line_tracker.recent_event() and self.last_frame_sent is not None :
fw_image = cv.CloneImage(self.last_frame_sent)
cv.Rectangle(fw_image, (0,0), (fw_image.width, fw_image.height),
cv.CV_RGB(255,0,255), 4)
cv.ShowImage(self.forward_window, fw_image)
else :
cv.ShowImage(self.forward_window, cv.QueryFrame(self.forwardcam))
self.previoustime = currenttime
# Handle keyboard input
return self.handle_keys()
def run (self) :
prev_time = time.time()
while self.repeat():
if time.time() - prev_time < 0.05:
time.sleep(time.time() - prev_time)
