async def handle_upload(request):
video = request.files.get('video')
form_data = request.form
async with aiofiles.open('./data/video.mp4', 'wb') as f:
await f.write(video.body)
blinks = [int(s) for s in form_data.get('blinks').split(',')]
b = BlinkDetector('shape_predictor_68_face_landmarks.dat',
'./data/video.mp4')
b.start()
print('Got Blink:{}, detected blinks: {}'.format(blinks, b.blinks))
status = 'success' if blinks == b.blinks else 'failure'
return response.json({'status': status})
def __init__(self,
saccade_detector=None,
blink_detector=None,
fixation_detector=None):
"""
Initialize the RecordingProcessor class
:param saccade_detector: the initialized object of SaccadeDetector class; if None, default init is used
:param blink_detector: the initialized object of BlinkDetector class; if None, default init is used
:param fixation_detector: the initialized object of FixationDetector class; if None, default init is used
"""
self._saccade_detector = saccade_detector if saccade_detector is not None else SaccadeDetector(
)
self._blink_detector = blink_detector if blink_detector is not None else BlinkDetector(
)
self._fixation_detector = fixation_detector if fixation_detector is not None else FixationDetector(
)
# loaders for different formats/sources of data
# should be all capital letters
self._format_loaders = {
'DSF': data_loaders.load_DSF_coord_as_arff_object,
'ARFF': data_loaders.load_ARFF_as_arff_object,
# This one is for arff files with already labelled eye movements, at least FIX and SACCADE types.
# It can be used either to load results of a different algorithm, or preprocessed data with partially
# labelled eye movements (i.e. your own saccade and fixation detector); is this type is chosen,
# nor saccade/blink/fixation detectors will be involved.
'LABELLED ARFF': data_loaders.load_ARFF_as_arff_object
}
def __init__(self, box, frame, shape_predictor, rect=None):
self.frame = frame
if rect is None:
self.rect = dlib.rectangle(box[0], box[1], box[2], box[3])
else:
self.rect = rect
self.shape_predictor = shape_predictor
self.counter = 0
self.id = BlinkDetector.registerBox(self)
self.left_open, self.right_open = self.__getEyesStatus()
self.is_previos_eye_closed = not (self.left_open and self.right_open)
self.open_counter = 0
def checkFrame(self):
blink_detector_response = BlinkDetector.detect(self.id, self.frame)
if blink_detector_response == 2: # Eyes are closed
print("Closed")
if not self.is_previos_eye_closed:
self.is_previos_eye_closed = True
self.counter += 1
self.open_counter = 0
elif blink_detector_response == 1 : # Eyes are opened
print("Opened")
self.open_counter += 1
if self.counter >= EYE_AR_THRESH and self.open_counter >= EYE_AR_THRESH: # Eyes are completely opened after blink
print("Real")
return True
self.counter = 0
else:
self.counter = 0
print("Can not decided")
return False
def detect_blinks(output_textfile, video_path):
ear_extractor = EarExtractor(video_path)
blink_detector = BlinkDetector()
number_of_frames = 0
while True:
(is_no_frame_left, is_face_detected, frame, left_eye, right_eye, ear) = ear_extractor.extract()
if is_no_frame_left:
print('no frame left')
print(number_of_frames)
break
if (is_face_detected == True):
number_of_frames = number_of_frames + 1 # number of frames that face is detected
bd_request = BlinkDetectorRequest(ear = ear, is_there_a_missing_ear = False)
retrieved_blinks = blink_detector.track_ears(bd_request)
if retrieved_blinks:
total_blinks = blink_detector.get_total_blinks()
blink_frame_freq = total_blinks / number_of_frames
for detected_blink in retrieved_blinks:
if (detected_blink.velocity > 0):
with open(output_file, 'ab') as f_handle:
f_handle.write(b'\n')
np.savetxt(f_handle,
[total_blinks, blink_frame_freq * 100,
detected_blink.amplitude, detected_blink.duration, detected_blink.velocity],
delimiter=', ', newline=' ',fmt='%.4f')
else:
bd_request = BlinkDetectorRequest(ear = 0, is_there_a_missing_ear = True)
blink_detector.track_ears(bd_request)
ear_extractor.release()
def __getEyesStatus(self):
return BlinkDetector.getEyesStatus(self.id, self.frame)
def run_detection(params):
"""
Run the entire detection pipeline with given parameters.
:param params: A two-level dictionary (just like create_parameters_from_args() would return).
The only required parameter is @params['GeneralArguments']['input_folder'], which should point
to a folder with raw gaze data. The data is assumed to be stored in the following way:
(1) for each movie (clip) there should be a separate subdirectory in the input_folder
(2) inside these subdirectories all the files with the extension of
@params['GeneralArguments']['gaze_extension'] (.coord by default) represent a recording for one
observer each.
If your data does not get loaded, maybe the appropriate data loader does not get called. You can
fix this (provided that the suitable data loader exists in data_loaders.py) by setting
@params['GeneralArguments']['input_data_type'] to the correct value (for correspondence see
the keys of RecordingProcessor._format_loaders).
To summarize, a minimalistic input to run detection with default parameters on your dataset
(let's assume you have converted the data to .arff format) would be:
run_detection({'GeneralArguments': {'input_folder': 'PATH/TO/YOUR/DATA/FOLDER',
'gaze_extension': '.arff'}})
:return: path to results folder
"""
# make a defaultdict out of @parameters so that we could always access its first-level keys
params_default_first_level = defaultdict(dict)
params_default_first_level.update(params)
params = params_default_first_level
verbose = params['GeneralArguments'].get('verbose', False)
out_folder = params['GeneralArguments'].get('output_folder')
if out_folder is None:
out_folder = tempfile.mkdtemp(prefix='sp_tool_')
warnings.warn('No output folder provided, using {}'.format(out_folder))
if verbose:
print >> sys.stderr, 'Outputs will be written to folder', out_folder
saccade_detector = SaccadeDetector(**params['SaccadeDetector'])
blink_detector = BlinkDetector(**params['BlinkDetector'])
fixation_detector = FixationDetector(**params['FixationDetector'])
recording_processor = RecordingProcessor(saccade_detector=saccade_detector,
blink_detector=blink_detector,
fixation_detector=fixation_detector)
sp_detector = SmoothPursuitDetector(**params['SmoothPursuitDetector'])
# The next lines deal with identifying the names of the video clips used for the eye tracking experiment.
# Can be initialized in various ways, here we just get all video paths be regex and cut off everything that
# is not needed.
#
#
in_folder = params['GeneralArguments'].get('input_folder')
if not in_folder:
raise ValueError('\'input_folder\' is a required parameter of the \'GeneralArguments\' group in @params!')
folder_names = sorted(glob.glob('{}/*/'.format(in_folder))) # getting all the folders of the input folder
# extract names from path
if not folder_names and verbose:
print >> sys.stderr, 'No subfolders found under "{}"'.format(in_folder)
folder_names = [os.path.splitext(os.path.basename(folder.rstrip('/')))[0] for folder in folder_names]
movies = params['GeneralArguments'].get('movies')
if movies: # not empty, restrict to these folders only
movies = set(movies)
folder_names = [fn for fn in folder_names if fn in movies]
if verbose:
print >> sys.stderr, 'Working with movies:', folder_names
# data files extension
gaze_pattern = params['GeneralArguments'].get('gaze_file_pattern', '*.coord')
if '*' not in gaze_pattern:
gaze_pattern = '*' + gaze_pattern
for movie in folder_names:
full_out_folder = '{}/{}/'.format(out_folder, movie)
if not os.path.exists(full_out_folder):
os.makedirs(full_out_folder)
if verbose:
print >> sys.stderr, 'Started processing for {},'.format(movie), 'results will appear in', full_out_folder
# The next lines load the data files of the recording with one particular movie.
# To do this, here we provide a regex that includes all the .{extension} files in the respective folder.
#
#
gaze_data_files = sorted(glob.glob('{}/{}/{}'.format(in_folder, movie, gaze_pattern)))
if len(gaze_data_files) == 0:
print >> sys.stderr, 'Found 0 files with this pattern: "{}". Omitting this directory.'.format(
'{}/{}/{}'.format(in_folder, movie, gaze_pattern)
)
continue
try:
# The next line loads the data, labels saccades, blinks and fixations.
gaze_points_list = recording_processor.load_multiple_recordings(
gaze_data_files, verbose=verbose, data_format=params['GeneralArguments'].get('input_data_type'))
# This will label the smooth pursuits
if verbose:
print >> sys.stderr, 'Saccades/blinks/fixations are detected, starting SP detection.'
classified_gaze_points = sp_detector.detect(gaze_points_list)
# Now just dump the resulting structure into .arff files in the respective subdirectory of the @out_folder
for file_name, arff_data in zip(gaze_data_files, classified_gaze_points):
output_file_name = os.path.splitext(os.path.basename(file_name))[0]
ArffHelper.dump(arff_data, open(
'{}/{}.arff'.format(full_out_folder, output_file_name), 'w')).close()
except Exception as e:
print >> sys.stderr, 'Had to skip {} due to an error "{}"'.format(movie, e.message)
return out_folder
def detect_drowsiness(self, video_path):
self.show_drowsy_video_window()
ear_extractor = EarExtractor(video_path)
blink_detector = BlinkDetector()
max_blink_per_sequence = 30
feature_count = 4
blink_sequence = np.zeros((max_blink_per_sequence, feature_count),
dtype=np.float32)
blink_count = 0
stride = 2
frameNth = 1
number_of_frames = 0
while True:
(is_no_frame_left, is_face_detected, frame, left_eye, right_eye,
ear) = ear_extractor.extract()
if is_no_frame_left:
break
self.drowsy_video_frame.update(frame)
self.update_gui()
if (is_face_detected == True):
number_of_frames = number_of_frames + 1
bd_request = BlinkDetectorRequest(ear=ear,
is_there_a_missing_ear=False)
retrieved_blinks = blink_detector.track_ears(bd_request)
if retrieved_blinks:
total_blinks = blink_detector.get_total_blinks()
blink_frame_freq = total_blinks / number_of_frames * 100
for detected_blink in retrieved_blinks:
freq = round(blink_frame_freq, 4)
amp = round(detected_blink.amplitude, 4)
dur = round(detected_blink.duration, 4)
vel = round(detected_blink.velocity, 4)
blink = np.array([freq, amp, dur, vel])
blink_sequence = rotate_left(blink_sequence, 1)
blink_sequence[max_blink_per_sequence - 1] = blink
blink_count = blink_count + 1
print('')
print('===========')
print('blink count: {0}'.format(blink_count))
print('')
if (self.is_cool_down_end()
and (blink_count >= max_blink_per_sequence)):
blink_sequence_list = np.array([blink_sequence])
results = self.do_detect_drowsiness(
blink_sequence_list)
else:
bd_request = BlinkDetectorRequest(ear=0,
is_there_a_missing_ear=True)
retrieved_blinks = blink_detector.track_ears(bd_request)
if (frameNth == 500):
self.show_drowsiness_alert()
elif (frameNth == 900):
self.hide_drowsiness_alert()
frameNth = 0
frameNth = frameNth + 1
def get_alert_state_features(self, video_path):
self.show_alert_video_window()
ear_extractor = EarExtractor(video_path)
blink_detector = BlinkDetector()
fps = ear_extractor.get_video_stream().get(cv2.CAP_PROP_FPS)
freqs = []
amps = []
durs = []
vels = []
number_of_frames = 0
count = 0
while True:
(is_no_frame_left, is_face_detected, frame, left_eye, right_eye,
ear) = ear_extractor.extract()
if is_no_frame_left:
print('end frame')
break
self.alert_video_frame.update(frame)
self.update_gui()
if (is_face_detected == True):
number_of_frames = number_of_frames + 1
bd_request = BlinkDetectorRequest(ear=ear,
is_there_a_missing_ear=False)
retrieved_blinks = blink_detector.track_ears(bd_request)
if retrieved_blinks:
total_blinks = blink_detector.get_total_blinks()
blink_frame_freq = total_blinks / number_of_frames * 100
for detected_blink in retrieved_blinks:
freqs.append(round(blink_frame_freq, 4))
amps.append(round(detected_blink.amplitude, 4))
durs.append(round(detected_blink.duration, 4))
vels.append(round(detected_blink.velocity, 4))
else:
bd_request = BlinkDetectorRequest(ear=0,
is_there_a_missing_ear=True)
retrieved_blinks = blink_detector.track_ears(bd_request)
count += 1
time_stamp = count / fps
if ((time_stamp) >= self.record_alert_end_time):
break
freqs = np.array(freqs)
amps = np.array(amps)
durs = np.array(durs)
vels = np.array(vels)
alert_mean_std = self.alert_mean_std
alert_mean_std.freq_mean = np.mean(freqs)
alert_mean_std.freq_stddev = np.std(freqs)
if alert_mean_std.freq_stddev == 0:
alert_mean_std.freq_stddev = 0.000001
alert_mean_std.amp_mean = np.mean(amps)
alert_mean_std.amp_stddev = np.std(amps)
if alert_mean_std.amp_stddev == 0:
alert_mean_std.amp_stddev = 0.000001
alert_mean_std.dur_mean = np.mean(durs)
alert_mean_std.dur_stddev = np.std(durs)
if alert_mean_std.dur_stddev == 0:
alert_mean_std.dur_stddev = 0.000001
alert_mean_std.vel_mean = np.mean(vels)
alert_mean_std.vel_stddev = np.std(vels)
if alert_mean_std.vel_stddev == 0:
alert_mean_std.vel_stddev = 0.000001
def blink_detector(output_textfile, video_path):
Q = Queue(maxsize=7)
ear_extractor = EarExtractor(video_path)
blink_detector = BlinkDetector()
blink_count = 0
number_of_frames = 0
while gui.cancel != True:
(is_no_frame_left, is_face_detected, frame, left_eye, right_eye,
ear) = ear_extractor.extract()
if is_no_frame_left:
print('no frame left')
print(number_of_frames)
break
Q.put(frame)
if (is_face_detected == True):
number_of_frames = number_of_frames + 1 # number of frames that face is detected
bd_request = BlinkDetectorRequest(ear=ear,
is_there_a_missing_ear=False)
retrieved_blinks = blink_detector.track_ears(bd_request)
if retrieved_blinks:
total_blinks = blink_detector.get_total_blinks()
blink_frame_freq = total_blinks / number_of_frames
blink_count = blink_count + len(retrieved_blinks)
print()
print('=============')
print("Blink count: {0}".format(blink_count))
print()
for detected_blink in retrieved_blinks:
if (detected_blink.velocity > 0):
with open(output_file, 'ab') as f_handle:
f_handle.write(b'\n')
np.savetxt(f_handle, [
total_blinks, blink_frame_freq * 100,
detected_blink.amplitude,
detected_blink.duration,
detected_blink.velocity
],
delimiter=', ',
newline=' ',
fmt='%.4f')
# compute the convex hull for the left and right eye, then
# visualize each of the eyes
left_eye_hull = cv2.convexHull(left_eye)
right_eye_hull = cv2.convexHull(right_eye)
cv2.drawContours(frame, [left_eye_hull], -1, (0, 255, 0), 1)
cv2.drawContours(frame, [right_eye_hull], -1, (0, 255, 0), 1)
if Q.full(
): #to make sure the frame of interest for the EAR vector is int the mid
gui.update_ear(blink_detector.get_current_ear_series())
frame_minus_7 = Q.get()
gui.update_video(frame_minus_7)
elif Q.full():
junk = Q.get()
key = cv2.waitKey(1) & 0xFF
# if the `q` key was pressed, break from the loop
if key != 0xFF:
break
else:
bd_request = BlinkDetectorRequest(ear=0,
is_there_a_missing_ear=True)
blink_detector.track_ears(bd_request)
while (Q.empty() != False):
frame_minus_7 = Q.get()
gui.update_video(frame_minus_7)
Q.queue.clear()
key = cv2.waitKey(1) & 0xFF
if key != 0xFF:
break
gui.update_gui()
# do a bit of cleanup
ear_extractor.release()
if __name__ == '__main__':
from test_fps import progress, capture
fourcc = 'MJPG'
width = 1920 * 2
height = 1080 * 2
fps = 5
log('waiting for availability')
cap = wait_for_format(fourcc, width, height, fps)
log('camera is available')
log('loading eye extractor and blink detector')
eye_extractor = EyeExtractor()
blink_detector = BlinkDetector()
log('loaded')
threshold = 0.1
hysteresis = Hysteresis()
try:
for i, img in enumerate(progress(capture(cap))):
sub = inner_square_crop(img)
eyes = eye_extractor(sub)
if eyes is None:
continue
blink = blink_detector(eyes)
state = hysteresis(blink < threshold)
def main():
args = vars(ap.parse_args())
# create frame counter
fps_counter = FPSCounter()
# total number of blinks
TOTAL = 0
# initialize dlib's face detector (HOG-based) and then create
# the facial landmark predictor
print("[INFO] loading facial landmark predictor...")
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(args["shape_predictor"])
# grab the indexes of the facial landmarks for the left and
# right eye, respectively
(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]
# start the video stream thread
print("[INFO] starting video stream thread...")
print("[INFO] print q to quit...")
if args['video'] == "camera":
vs = VideoStream(src=0).start()
vs.stream.set(cv2.CAP_PROP_FPS, 15)
fileStream = False
else:
vs = FileVideoStream(args["video"]).start()
fileStream = True
fps = vs.stream.get(cv2.CAP_PROP_FPS)
# create dataloggers
datalogger = DataLogger(columns=['ear', 'adr'])
# blink detector
blink_detector = BlinkDetector(time_window=5,
plot=args['graph'],
frame_delay=10)
# loop over frames from the video stream
frame_cnt = 0
INIT_TIME = None
while True:
# if this is a file video stream, then we need to check if
# there any more frames left in the buffer to process
if fileStream and not vs.more():
break
# get timestamp
if fileStream:
timestamp = frame_cnt / fps
else:
if INIT_TIME is None:
INIT_TIME = time.time()
timestamp = time.time() - INIT_TIME
fps = fps_counter.tick()
# get the new frame
frame = vs.read()
frame_cnt += 1
if frame is None:
break
frame = imutils.resize(frame, width=450)
# it, and convert it to grayscale channels)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# detect faces in the grayscale frame
rects = detector(gray, 0)
# loop over the face detections
for rect in rects:
# determine the facial landmarks for the face region, then
# convert the facial landmark (x, y)-coordinates to a NumPy
# array
shape = predictor(gray, rect)
shape = face_utils.shape_to_np(shape)
# extract the left and right eye coordinates, then use the
# coordinates to compute the eye aspect ratio for both eyes
leftEye = shape[lStart:lEnd]
rightEye = shape[rStart:rEnd]
leftEAR = eye_aspect_ratio(leftEye)
rightEAR = eye_aspect_ratio(rightEye)
# compute the area-over-distance metric
adr = AreaDistanceRatio.compute(leftEye, rightEye)
# log ADR
datalogger.log(adr, 'adr', timestamp)
# average the eye aspect ratio together for both eyes
ear = (leftEAR + rightEAR) / 2.0
# log EAR
datalogger.log(ear, 'ear', timestamp)
# compute the convex hull for the left and right eye, then
# visualize each of the eyes
leftEyeHull = cv2.convexHull(leftEye)
rightEyeHull = cv2.convexHull(rightEye)
cv2.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1)
cv2.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1)
# send new data to blink detector and check if it detected new blinks
blink_detector.send(adr, timestamp)
blink = blink_detector.get_blink()
if blink is not None:
blink_time, blink_dur = blink
TOTAL += 1
print(f"[BLINK] time: {blink_time:.2f} dur: {blink_dur:.2f}")
# draw the total number of blinks on the frame along with
# the computed eye aspect ratio for the frame
cv2.putText(frame, "Blinks: {}".format(TOTAL), (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
cv2.putText(frame, "ADR: {:.2f}".format(ear), (300, 30),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
cv2.putText(frame, "FPS: {:.2f}".format(fps), (300, 60),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
# show the frame
cv2.imshow("Frame", frame)
key = cv2.waitKey(1) & 0xFF
# if the `q` key was pressed, break from the loop
if key == ord("q"):
break
# save datafile
output_file = args['output_file']
if output_file == 'ask':
output_file = input("Enter filename to save: ")
if output_file is not None:
datalogger.save(output_file)
# do a bit of cleanup
cv2.destroyAllWindows()
vs.stop()