Beispiel #1
0
    async def handle_upload(request):
        video = request.files.get('video')
        form_data = request.form
        async with aiofiles.open('./data/video.mp4', 'wb') as f:
            await f.write(video.body)

        blinks = [int(s) for s in form_data.get('blinks').split(',')]
        b = BlinkDetector('shape_predictor_68_face_landmarks.dat',
                          './data/video.mp4')
        b.start()
        print('Got Blink:{}, detected blinks: {}'.format(blinks, b.blinks))
        status = 'success' if blinks == b.blinks else 'failure'
        return response.json({'status': status})
Beispiel #2
0
    def __init__(self,
                 saccade_detector=None,
                 blink_detector=None,
                 fixation_detector=None):
        """
        Initialize the RecordingProcessor class

        :param saccade_detector: the initialized object of SaccadeDetector class; if None, default init is used
        :param blink_detector: the initialized object of BlinkDetector class; if None, default init is used
        :param fixation_detector: the initialized object of FixationDetector class; if None, default init is used
        """
        self._saccade_detector = saccade_detector if saccade_detector is not None else SaccadeDetector(
        )
        self._blink_detector = blink_detector if blink_detector is not None else BlinkDetector(
        )
        self._fixation_detector = fixation_detector if fixation_detector is not None else FixationDetector(
        )

        # loaders for different formats/sources of data
        # should be all capital letters
        self._format_loaders = {
            'DSF': data_loaders.load_DSF_coord_as_arff_object,
            'ARFF': data_loaders.load_ARFF_as_arff_object,
            # This one is for arff files with already labelled eye movements, at least FIX and SACCADE types.
            # It can be used either to load results of a different algorithm, or preprocessed data with partially
            # labelled eye movements (i.e. your own saccade and fixation detector); is this type is chosen,
            # nor saccade/blink/fixation detectors will be involved.
            'LABELLED ARFF': data_loaders.load_ARFF_as_arff_object
        }
 def __init__(self, box, frame, shape_predictor, rect=None):
     self.frame = frame
     if rect is None:
         self.rect = dlib.rectangle(box[0], box[1], box[2], box[3])
     else:
         self.rect = rect
     self.shape_predictor = shape_predictor
     self.counter = 0
     self.id = BlinkDetector.registerBox(self)
     self.left_open, self.right_open = self.__getEyesStatus()
     self.is_previos_eye_closed = not (self.left_open and self.right_open)
     self.open_counter = 0
 def checkFrame(self):
     blink_detector_response = BlinkDetector.detect(self.id, self.frame)
     if blink_detector_response == 2: # Eyes are closed
         print("Closed")
         if not self.is_previos_eye_closed:
             self.is_previos_eye_closed = True
         self.counter += 1
         self.open_counter = 0
     elif blink_detector_response == 1 : # Eyes are opened
         print("Opened")
         self.open_counter += 1
         if self.counter >= EYE_AR_THRESH and self.open_counter >= EYE_AR_THRESH: # Eyes are completely opened after blink
             print("Real")
             return True
         self.counter = 0   
     else:
         self.counter = 0
         print("Can not decided")
     return False
def detect_blinks(output_textfile, video_path):
    ear_extractor = EarExtractor(video_path)
    blink_detector = BlinkDetector()

    number_of_frames = 0
    while True:
        (is_no_frame_left, is_face_detected, frame, left_eye, right_eye, ear) = ear_extractor.extract()
        if is_no_frame_left:
            print('no frame left')
            print(number_of_frames)
            break


        if (is_face_detected == True):
            
            number_of_frames = number_of_frames + 1  # number of frames that face is detected

            bd_request = BlinkDetectorRequest(ear = ear, is_there_a_missing_ear = False)
            retrieved_blinks = blink_detector.track_ears(bd_request)

            if retrieved_blinks:
                total_blinks = blink_detector.get_total_blinks()
                blink_frame_freq = total_blinks / number_of_frames
                
                for detected_blink in retrieved_blinks:
                    if (detected_blink.velocity > 0):
                        with open(output_file, 'ab') as f_handle:
                            f_handle.write(b'\n')
                            np.savetxt(f_handle,
                                [total_blinks, blink_frame_freq * 100,
                                    detected_blink.amplitude, detected_blink.duration, detected_blink.velocity], 
                                delimiter=', ', newline=' ',fmt='%.4f')
        
        else:
            bd_request = BlinkDetectorRequest(ear = 0, is_there_a_missing_ear = True)
            blink_detector.track_ears(bd_request)

    ear_extractor.release()
 def __getEyesStatus(self):
     return BlinkDetector.getEyesStatus(self.id, self.frame)
Beispiel #7
0
def run_detection(params):
    """
    Run the entire detection pipeline with given parameters.
    :param params: A two-level dictionary (just like create_parameters_from_args() would return).
                   The only required parameter is @params['GeneralArguments']['input_folder'], which should point
                   to a folder with raw gaze data. The data is assumed to be stored in the following way:
                     (1) for each movie (clip) there should be a separate subdirectory in the input_folder
                     (2) inside these subdirectories all the files with the extension of
                     @params['GeneralArguments']['gaze_extension'] (.coord by default) represent a recording for one
                     observer each.
                   If your data does not get loaded, maybe the appropriate data loader does not get called. You can
                   fix this (provided that the suitable data loader exists in data_loaders.py) by setting
                   @params['GeneralArguments']['input_data_type'] to the correct value (for correspondence see
                   the keys of RecordingProcessor._format_loaders).

                   To summarize, a minimalistic input to run detection with default parameters on your dataset
                   (let's assume you have converted the data to .arff format) would be:

                   run_detection({'GeneralArguments': {'input_folder': 'PATH/TO/YOUR/DATA/FOLDER',
                                                       'gaze_extension': '.arff'}})
    :return: path to results folder
    """
    # make a defaultdict  out of @parameters so that we could always access its first-level keys
    params_default_first_level = defaultdict(dict)
    params_default_first_level.update(params)
    params = params_default_first_level

    verbose = params['GeneralArguments'].get('verbose', False)

    out_folder = params['GeneralArguments'].get('output_folder')
    if out_folder is None:
        out_folder = tempfile.mkdtemp(prefix='sp_tool_')
        warnings.warn('No output folder provided, using {}'.format(out_folder))
    if verbose:
        print >> sys.stderr, 'Outputs will be written to folder', out_folder

    saccade_detector = SaccadeDetector(**params['SaccadeDetector'])
    blink_detector = BlinkDetector(**params['BlinkDetector'])
    fixation_detector = FixationDetector(**params['FixationDetector'])

    recording_processor = RecordingProcessor(saccade_detector=saccade_detector,
                                             blink_detector=blink_detector,
                                             fixation_detector=fixation_detector)

    sp_detector = SmoothPursuitDetector(**params['SmoothPursuitDetector'])

    # The next lines deal with identifying the names of the video clips used for the eye tracking experiment.
    # Can be initialized in various ways, here we just get all video paths be regex and cut off everything that
    # is not needed.
    #
    #
    in_folder = params['GeneralArguments'].get('input_folder')
    if not in_folder:
        raise ValueError('\'input_folder\' is a required parameter of the \'GeneralArguments\' group in @params!')
    folder_names = sorted(glob.glob('{}/*/'.format(in_folder)))  # getting all the folders of the input folder
    # extract names from path
    if not folder_names and verbose:
        print >> sys.stderr, 'No subfolders found under "{}"'.format(in_folder)
    folder_names = [os.path.splitext(os.path.basename(folder.rstrip('/')))[0] for folder in folder_names]

    movies = params['GeneralArguments'].get('movies')
    if movies:  # not empty, restrict to these folders only
        movies = set(movies)
        folder_names = [fn for fn in folder_names if fn in movies]

    if verbose:
        print >> sys.stderr, 'Working with movies:', folder_names

    # data files extension
    gaze_pattern = params['GeneralArguments'].get('gaze_file_pattern', '*.coord')
    if '*' not in gaze_pattern:
        gaze_pattern = '*' + gaze_pattern

    for movie in folder_names:
        full_out_folder = '{}/{}/'.format(out_folder, movie)
        if not os.path.exists(full_out_folder):
            os.makedirs(full_out_folder)
        if verbose:
            print >> sys.stderr, 'Started processing for {},'.format(movie), 'results will appear in', full_out_folder

        # The next lines load the data files of the recording with one particular movie.
        # To do this, here we provide a regex that includes all the .{extension} files in the respective folder.
        #
        #
        gaze_data_files = sorted(glob.glob('{}/{}/{}'.format(in_folder, movie, gaze_pattern)))
        if len(gaze_data_files) == 0:
            print >> sys.stderr, 'Found 0 files with this pattern: "{}". Omitting this directory.'.format(
                '{}/{}/{}'.format(in_folder, movie, gaze_pattern)
            )
            continue
        try:
            # The next line loads the data, labels saccades, blinks and fixations.
            gaze_points_list = recording_processor.load_multiple_recordings(
                gaze_data_files, verbose=verbose, data_format=params['GeneralArguments'].get('input_data_type'))
            # This will label the smooth pursuits
            if verbose:
                print >> sys.stderr, 'Saccades/blinks/fixations are detected, starting SP detection.'
            classified_gaze_points = sp_detector.detect(gaze_points_list)

            # Now just dump the resulting structure into .arff files in the respective subdirectory of the @out_folder
            for file_name, arff_data in zip(gaze_data_files, classified_gaze_points):
                output_file_name = os.path.splitext(os.path.basename(file_name))[0]
                ArffHelper.dump(arff_data, open(
                    '{}/{}.arff'.format(full_out_folder, output_file_name), 'w')).close()
        except Exception as e:
            print >> sys.stderr, 'Had to skip {} due to an error "{}"'.format(movie, e.message)
    return out_folder
Beispiel #8
0
    def detect_drowsiness(self, video_path):
        self.show_drowsy_video_window()

        ear_extractor = EarExtractor(video_path)
        blink_detector = BlinkDetector()

        max_blink_per_sequence = 30
        feature_count = 4
        blink_sequence = np.zeros((max_blink_per_sequence, feature_count),
                                  dtype=np.float32)
        blink_count = 0
        stride = 2

        frameNth = 1
        number_of_frames = 0
        while True:
            (is_no_frame_left, is_face_detected, frame, left_eye, right_eye,
             ear) = ear_extractor.extract()
            if is_no_frame_left:
                break

            self.drowsy_video_frame.update(frame)
            self.update_gui()

            if (is_face_detected == True):
                number_of_frames = number_of_frames + 1

                bd_request = BlinkDetectorRequest(ear=ear,
                                                  is_there_a_missing_ear=False)
                retrieved_blinks = blink_detector.track_ears(bd_request)

                if retrieved_blinks:
                    total_blinks = blink_detector.get_total_blinks()
                    blink_frame_freq = total_blinks / number_of_frames * 100

                    for detected_blink in retrieved_blinks:
                        freq = round(blink_frame_freq, 4)
                        amp = round(detected_blink.amplitude, 4)
                        dur = round(detected_blink.duration, 4)
                        vel = round(detected_blink.velocity, 4)

                        blink = np.array([freq, amp, dur, vel])
                        blink_sequence = rotate_left(blink_sequence, 1)
                        blink_sequence[max_blink_per_sequence - 1] = blink
                        blink_count = blink_count + 1

                        print('')
                        print('===========')
                        print('blink count: {0}'.format(blink_count))
                        print('')
                        if (self.is_cool_down_end()
                                and (blink_count >= max_blink_per_sequence)):
                            blink_sequence_list = np.array([blink_sequence])
                            results = self.do_detect_drowsiness(
                                blink_sequence_list)

            else:
                bd_request = BlinkDetectorRequest(ear=0,
                                                  is_there_a_missing_ear=True)
                retrieved_blinks = blink_detector.track_ears(bd_request)

            if (frameNth == 500):
                self.show_drowsiness_alert()
            elif (frameNth == 900):
                self.hide_drowsiness_alert()
                frameNth = 0

            frameNth = frameNth + 1
Beispiel #9
0
    def get_alert_state_features(self, video_path):
        self.show_alert_video_window()

        ear_extractor = EarExtractor(video_path)
        blink_detector = BlinkDetector()

        fps = ear_extractor.get_video_stream().get(cv2.CAP_PROP_FPS)

        freqs = []
        amps = []
        durs = []
        vels = []

        number_of_frames = 0
        count = 0
        while True:
            (is_no_frame_left, is_face_detected, frame, left_eye, right_eye,
             ear) = ear_extractor.extract()
            if is_no_frame_left:
                print('end frame')
                break

            self.alert_video_frame.update(frame)
            self.update_gui()

            if (is_face_detected == True):
                number_of_frames = number_of_frames + 1

                bd_request = BlinkDetectorRequest(ear=ear,
                                                  is_there_a_missing_ear=False)
                retrieved_blinks = blink_detector.track_ears(bd_request)

                if retrieved_blinks:
                    total_blinks = blink_detector.get_total_blinks()
                    blink_frame_freq = total_blinks / number_of_frames * 100

                    for detected_blink in retrieved_blinks:
                        freqs.append(round(blink_frame_freq, 4))
                        amps.append(round(detected_blink.amplitude, 4))
                        durs.append(round(detected_blink.duration, 4))
                        vels.append(round(detected_blink.velocity, 4))
            else:
                bd_request = BlinkDetectorRequest(ear=0,
                                                  is_there_a_missing_ear=True)
                retrieved_blinks = blink_detector.track_ears(bd_request)

            count += 1
            time_stamp = count / fps
            if ((time_stamp) >= self.record_alert_end_time):
                break

        freqs = np.array(freqs)
        amps = np.array(amps)
        durs = np.array(durs)
        vels = np.array(vels)

        alert_mean_std = self.alert_mean_std

        alert_mean_std.freq_mean = np.mean(freqs)
        alert_mean_std.freq_stddev = np.std(freqs)
        if alert_mean_std.freq_stddev == 0:
            alert_mean_std.freq_stddev = 0.000001

        alert_mean_std.amp_mean = np.mean(amps)
        alert_mean_std.amp_stddev = np.std(amps)
        if alert_mean_std.amp_stddev == 0:
            alert_mean_std.amp_stddev = 0.000001

        alert_mean_std.dur_mean = np.mean(durs)
        alert_mean_std.dur_stddev = np.std(durs)
        if alert_mean_std.dur_stddev == 0:
            alert_mean_std.dur_stddev = 0.000001

        alert_mean_std.vel_mean = np.mean(vels)
        alert_mean_std.vel_stddev = np.std(vels)
        if alert_mean_std.vel_stddev == 0:
            alert_mean_std.vel_stddev = 0.000001
    def blink_detector(output_textfile, video_path):

        Q = Queue(maxsize=7)

        ear_extractor = EarExtractor(video_path)
        blink_detector = BlinkDetector()

        blink_count = 0

        number_of_frames = 0
        while gui.cancel != True:
            (is_no_frame_left, is_face_detected, frame, left_eye, right_eye,
             ear) = ear_extractor.extract()
            if is_no_frame_left:
                print('no frame left')
                print(number_of_frames)
                break

            Q.put(frame)

            if (is_face_detected == True):

                number_of_frames = number_of_frames + 1  # number of frames that face is detected

                bd_request = BlinkDetectorRequest(ear=ear,
                                                  is_there_a_missing_ear=False)
                retrieved_blinks = blink_detector.track_ears(bd_request)

                if retrieved_blinks:
                    total_blinks = blink_detector.get_total_blinks()
                    blink_frame_freq = total_blinks / number_of_frames

                    blink_count = blink_count + len(retrieved_blinks)
                    print()
                    print('=============')
                    print("Blink count: {0}".format(blink_count))
                    print()

                    for detected_blink in retrieved_blinks:
                        if (detected_blink.velocity > 0):
                            with open(output_file, 'ab') as f_handle:
                                f_handle.write(b'\n')
                                np.savetxt(f_handle, [
                                    total_blinks, blink_frame_freq * 100,
                                    detected_blink.amplitude,
                                    detected_blink.duration,
                                    detected_blink.velocity
                                ],
                                           delimiter=', ',
                                           newline=' ',
                                           fmt='%.4f')

                # compute the convex hull for the left and right eye, then
                # visualize each of the eyes
                left_eye_hull = cv2.convexHull(left_eye)
                right_eye_hull = cv2.convexHull(right_eye)
                cv2.drawContours(frame, [left_eye_hull], -1, (0, 255, 0), 1)
                cv2.drawContours(frame, [right_eye_hull], -1, (0, 255, 0), 1)

                if Q.full(
                ):  #to make sure the frame of interest for the EAR vector is int the mid
                    gui.update_ear(blink_detector.get_current_ear_series())
                    frame_minus_7 = Q.get()
                    gui.update_video(frame_minus_7)

                elif Q.full():
                    junk = Q.get()

                key = cv2.waitKey(1) & 0xFF

                # if the `q` key was pressed, break from the loop
                if key != 0xFF:
                    break

            else:
                bd_request = BlinkDetectorRequest(ear=0,
                                                  is_there_a_missing_ear=True)
                blink_detector.track_ears(bd_request)

                while (Q.empty() != False):
                    frame_minus_7 = Q.get()
                    gui.update_video(frame_minus_7)

                Q.queue.clear()

                key = cv2.waitKey(1) & 0xFF

                if key != 0xFF:
                    break

            gui.update_gui()

        # do a bit of cleanup
        ear_extractor.release()
Beispiel #11
0
if __name__ == '__main__':

    from test_fps import progress, capture

    fourcc = 'MJPG'
    width = 1920 * 2
    height = 1080 * 2
    fps = 5

    log('waiting for availability')
    cap = wait_for_format(fourcc, width, height, fps)
    log('camera is available')

    log('loading eye extractor and blink detector')
    eye_extractor = EyeExtractor()
    blink_detector = BlinkDetector()
    log('loaded')

    threshold = 0.1
    hysteresis = Hysteresis()

    try:
        for i, img in enumerate(progress(capture(cap))):

            sub = inner_square_crop(img)
            eyes = eye_extractor(sub)
            if eyes is None:
                continue
            blink = blink_detector(eyes)

            state = hysteresis(blink < threshold)
def main():
    args = vars(ap.parse_args())

    # create frame counter
    fps_counter = FPSCounter()

    # total number of blinks
    TOTAL = 0

    # initialize dlib's face detector (HOG-based) and then create
    # the facial landmark predictor
    print("[INFO] loading facial landmark predictor...")
    detector = dlib.get_frontal_face_detector()
    predictor = dlib.shape_predictor(args["shape_predictor"])

    # grab the indexes of the facial landmarks for the left and
    # right eye, respectively
    (lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
    (rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]

    # start the video stream thread
    print("[INFO] starting video stream thread...")
    print("[INFO] print q to quit...")
    if args['video'] == "camera":
        vs = VideoStream(src=0).start()
        vs.stream.set(cv2.CAP_PROP_FPS, 15)
        fileStream = False
    else:
        vs = FileVideoStream(args["video"]).start()
        fileStream = True
        fps = vs.stream.get(cv2.CAP_PROP_FPS)

    # create dataloggers
    datalogger = DataLogger(columns=['ear', 'adr'])

    # blink detector
    blink_detector = BlinkDetector(time_window=5,
                                   plot=args['graph'],
                                   frame_delay=10)

    # loop over frames from the video stream
    frame_cnt = 0
    INIT_TIME = None
    while True:
        # if this is a file video stream, then we need to check if
        # there any more frames left in the buffer to process
        if fileStream and not vs.more():
            break

        # get timestamp
        if fileStream:
            timestamp = frame_cnt / fps
        else:
            if INIT_TIME is None:
                INIT_TIME = time.time()
            timestamp = time.time() - INIT_TIME
            fps = fps_counter.tick()

        # get the new frame
        frame = vs.read()
        frame_cnt += 1
        if frame is None:
            break

        frame = imutils.resize(frame, width=450)
        # it, and convert it to grayscale channels)
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

        # detect faces in the grayscale frame
        rects = detector(gray, 0)

        # loop over the face detections
        for rect in rects:
            # determine the facial landmarks for the face region, then
            # convert the facial landmark (x, y)-coordinates to a NumPy
            # array

            shape = predictor(gray, rect)
            shape = face_utils.shape_to_np(shape)

            # extract the left and right eye coordinates, then use the
            # coordinates to compute the eye aspect ratio for both eyes
            leftEye = shape[lStart:lEnd]
            rightEye = shape[rStart:rEnd]
            leftEAR = eye_aspect_ratio(leftEye)
            rightEAR = eye_aspect_ratio(rightEye)

            # compute the area-over-distance metric
            adr = AreaDistanceRatio.compute(leftEye, rightEye)
            # log ADR
            datalogger.log(adr, 'adr', timestamp)

            # average the eye aspect ratio together for both eyes
            ear = (leftEAR + rightEAR) / 2.0
            # log EAR
            datalogger.log(ear, 'ear', timestamp)

            # compute the convex hull for the left and right eye, then
            # visualize each of the eyes
            leftEyeHull = cv2.convexHull(leftEye)
            rightEyeHull = cv2.convexHull(rightEye)
            cv2.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1)
            cv2.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1)

            # send new data to blink detector and check if it detected new blinks
            blink_detector.send(adr, timestamp)
            blink = blink_detector.get_blink()
            if blink is not None:
                blink_time, blink_dur = blink
                TOTAL += 1
                print(f"[BLINK] time: {blink_time:.2f}  dur: {blink_dur:.2f}")

            # draw the total number of blinks on the frame along with
            # the computed eye aspect ratio for the frame
            cv2.putText(frame, "Blinks: {}".format(TOTAL), (10, 30),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
            cv2.putText(frame, "ADR: {:.2f}".format(ear), (300, 30),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
            cv2.putText(frame, "FPS: {:.2f}".format(fps), (300, 60),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)

        # show the frame
        cv2.imshow("Frame", frame)
        key = cv2.waitKey(1) & 0xFF

        # if the `q` key was pressed, break from the loop
        if key == ord("q"):
            break

    # save datafile
    output_file = args['output_file']
    if output_file == 'ask':
        output_file = input("Enter filename to save: ")
    if output_file is not None:
        datalogger.save(output_file)

    # do a bit of cleanup
    cv2.destroyAllWindows()
    vs.stop()