Esempio n. 1
0
    def _perform_stop(self):
        current_mode = self.__current_mode
        pupil_list = self.__pupil_list
        ref_list = self.__ref_list

        logger.info(f"Stopping  {current_mode.label}")
        audio.tink()

        ### Set the calibration choreography state

        self.__is_active = False
        self.__ref_list = []
        self.__pupil_list = []

        ### Set the calibration choreography UI

        self.status_text = None

        # Show all buttons for all the modes
        for mode in list(ChoreographyMode):
            self.__toggle_mode_button_visibility(is_visible=True, mode=mode)

        ### Call relevant callbacks

        self.on_choreography_stopped(mode=current_mode)

        self.on_choreography_successfull(
            mode=current_mode, pupil_list=pupil_list, ref_list=ref_list
        )
Esempio n. 2
0
    def _perform_start(self):
        if self.__is_active:
            logger.debug(
                "[PROGRAMMING ERROR] Called _perform_start on an already active "
                "calibration choreography."
            )
            return

        current_mode = self.__current_mode

        logger.info(f"Starting  {current_mode.label}")
        audio.tink()

        ### Set the calibration choreography state

        self.__is_active = True
        self.__ref_list = []
        self.__pupil_list = []

        ### Set the calibration choreography UI

        # Hide all buttons for the mode buttons that are not currently used
        for mode in list(ChoreographyMode):
            if self.__current_mode != mode:
                self.__toggle_mode_button_visibility(is_visible=False, mode=mode)

        ### Call relevant callbacks

        self.on_choreography_started(mode=current_mode)
    def recent_events(self, events):
        frame = events.get('frame')
        if self.active and frame:
            gray_img = frame.gray

            if self.clicks_to_close <= 0:
                self.stop()
                return

            # Update the marker
            self.markers = self.circle_tracker.update(gray_img)

            self.stop_marker_found = False
            if len(self.markers):
                # Set the pos to be the center of the first detected marker
                marker_pos = self.markers[0]['img_pos']
                self.pos = self.markers[0]['norm_pos']
                # Check if there are stop markers
                for marker in self.markers:
                    if marker['marker_type'] == 'Stop':
                        self.auto_stop += 1
                        self.stop_marker_found = True
                        break
            else:
                self.pos = None  # indicate that no reference is detected

            if self.stop_marker_found is False:
                self.auto_stop = 0

            # Check if there are more than one markers
            if len(self.markers) > 1:
                audio.tink()
                logger.warning("{} markers detected. Please remove all the other markers".format(len(self.markers)))

            # only save a valid ref position if within sample window of calibraiton routine
            on_position = self.lead_in < self.screen_marker_state

            if on_position and len(self.markers) and not self.stop_marker_found:
                ref = {}
                ref["norm_pos"] = self.pos
                ref["screen_pos"] = marker_pos
                ref["timestamp"] = frame.timestamp
                self.ref_list.append(ref)

            # always save pupil positions
            self.pupil_list.extend(events['pupil_positions'])

            # Animate the screen marker
            if len(self.markers) or not on_position:
                self.screen_marker_state += 1

            # Stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()

            # use np.arrays for per element wise math
            self.on_position = on_position
        if self._window:
            self.gl_display_in_window()
    def recent_events(self, events):
        frame = events.get('frame')
        if self.active and frame:
            recent_pupil_positions = events['pupil_positions']
            gray_img = frame.gray

            if self.clicks_to_close <= 0:
                self.stop()
                return

            # Update the marker
            self.markers = self.circle_tracker.update(gray_img)
            # Screen marker takes only Ref marker
            self.markers = [
                marker for marker in self.markers
                if marker['marker_type'] == 'Ref'
            ]

            if len(self.markers):
                # Set the pos to be the center of the first detected marker
                marker_pos = self.markers[0]['img_pos']
                self.pos = self.markers[0]['norm_pos']
            else:
                self.pos = None  # indicate that no reference is detected

            # Check if there are more than one markers
            if len(self.markers) > 1:
                audio.tink()
                logger.warning(
                    "{} markers detected. Please remove all the other markers".
                    format(len(self.markers)))

            # only save a valid ref position if within sample window of calibraiton routine
            on_position = self.lead_in < self.screen_marker_state

            if on_position and len(self.markers):
                ref = {}
                ref["norm_pos"] = self.pos
                ref["screen_pos"] = marker_pos
                ref["timestamp"] = frame.timestamp
                self.ref_list.append(ref)

            # always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            # Animate the screen marker
            if len(self.markers) or not on_position:
                self.screen_marker_state += 1

            # use np.arrays for per element wise math
            self.on_position = on_position
        if self._window:
            self.gl_display_in_window()
Esempio n. 5
0
    def _perform_stop(self):
        if not self.__is_active:
            logger.debug(
                "[PROGRAMMING ERROR] Called _perform_stop on an already inactive "
                "calibration choreography."
            )
            return

        if self.g_pool.app == "capture":
            # Reset the main window size to trigger a redraw with correct size and scale
            # Only run in Capture to fix https://github.com/pupil-labs/pupil/issues/2119
            self.g_pool.trigger_main_window_redraw()

        current_mode = self.__current_mode
        pupil_list = self.__pupil_list
        ref_list = self.__ref_list

        logger.info(f"Stopping  {current_mode.label}")
        audio.tink()

        ### Set the calibration choreography state

        self.__is_active = False
        self.__ref_list = []
        self.__pupil_list = []

        ### Set the calibration choreography UI

        self.status_text = None

        # Show all buttons for all the modes
        for mode in list(ChoreographyMode):
            self.__toggle_mode_button_visibility(is_visible=True, mode=mode)

        ### Call relevant callbacks

        self.on_choreography_stopped(mode=current_mode)

        self.on_choreography_successfull(
            mode=current_mode, pupil_list=pupil_list, ref_list=ref_list
        )
Esempio n. 6
0
    def update(self,frame,events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        if self.active:
            recent_pupil_positions = events['pupil_positions']

            gray_img  = frame.gray

            if self.world_size is None:
                self.world_size = frame.width,frame.height

            self.candidate_ellipses = get_candidate_ellipses(gray_img,
                                                            area_threshold=self.area_threshold,
                                                            dist_threshold=self.dist_threshold,
                                                            min_ring_count=5,
                                                            visual_debug=self.show_edges)

            if len(self.candidate_ellipses) > 0:
                self.detected = True
                marker_pos = self.candidate_ellipses[0][0]
                self.pos = normalize(marker_pos,(frame.width,frame.height),flip_y=True)


            else:
                self.detected = False
                self.pos = None #indicate that no reference is detected


            # center dark or white?
            if self.detected:
                second_ellipse =  self.candidate_ellipses[1]
                col_slice = int(second_ellipse[0][0]-second_ellipse[1][0]/2),int(second_ellipse[0][0]+second_ellipse[1][0]/2)
                row_slice = int(second_ellipse[0][1]-second_ellipse[1][1]/2),int(second_ellipse[0][1]+second_ellipse[1][1]/2)
                marker_gray = gray_img[slice(*row_slice),slice(*col_slice)]
                avg = cv2.mean(marker_gray)[0] #CV2 fn return has changed!
                center = marker_gray[second_ellipse[1][1]/2,second_ellipse[1][0]/2]
                rel_shade = center-avg

                #auto_stop logic
                if rel_shade > 30:
                    #bright marker center found
                    self.auto_stop +=1
                    self.stop_marker_found = True

                else:
                    self.auto_stop = 0
                    self.stop_marker_found = False


            #tracking logic
            if self.detected and not self.stop_marker_found:
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother*(pos-smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0])+abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother*(new_vel-self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos-np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0])+abs(sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if not self.counter:

                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug("Steady marker found. Starting to sample %s datapoints" %self.counter_max)
                        self.counter = self.counter_max

                if self.counter:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.debug("Marker moved to quickly: Aborted sample. Sampled %s datapoints. Looking for steady marker again."%(self.counter_max-self.counter))
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if self.counter == 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug("Sampled %s datapoints. Stopping to sample. Looking for steady marker again."%self.counter_max)


            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.g_pool.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if self.counter:
                if self.detected:
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'



            #stop if autostop condition is satisfied:
            if self.auto_stop >=self.auto_stop_max:
                self.auto_stop = 0
                self.stop()


        else:
            pass
Esempio n. 7
0
    def update(self,frame,events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        if self.active:
            recent_pupil_positions = events['pupil_positions']

            gray_img  = frame.gray

            if self.world_size is None:
                self.world_size = frame.width,frame.height

            self.markers = find_concetric_circles(gray_img,min_ring_count=3)

            if len(self.markers) > 0:
                self.detected = True
                marker_pos = self.markers[0][0][0] #first marker, innermost ellipse, center
                self.pos = normalize(marker_pos,(frame.width,frame.height),flip_y=True)



            else:
                self.detected = False
                self.pos = None #indicate that no reference is detected



            #tracking logic
            if self.detected:
                self.auto_stop +=1
                self.stop_marker_found = True

                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother*(pos-smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0])+abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother*(new_vel-self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos-np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0])+abs(sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if not self.counter:

                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug("Steady marker found. Starting to sample {} datapoints".format(self.counter_max))
                        self.counter = self.counter_max

                if self.counter:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.warning("Marker moved to quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again.".format((self.counter_max-self.counter)))
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = denormalize(self.pos,(frame.width,frame.height),flip_y=True)
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if self.counter == 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug("Sampled {} datapoints. Stopping to sample. Looking for steady marker again.".format(self.counter_max))

            #always save pupil positions
            for pt in events.get('gaze_positions',[]):
                if pt['confidence'] > self.pupil_confidence_threshold:
                    #we add an id for the calibration preprocess data to work as is usually expects pupil data.
                    pt['id'] = 0
                    self.gaze_list.append(pt)

            if self.counter:
                if self.detected:
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'

            # stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()
        else:
            pass
    def recent_events(self, events):
        frame = events.get("frame")
        if self.active and frame:
            gray_img = frame.gray

            if self.clicks_to_close <= 0:
                self.stop()
                return

            # Update the marker
            self.markers = self.circle_tracker.update(gray_img)
            # Screen marker takes only Ref marker
            self.markers = [
                marker for marker in self.markers
                if marker["marker_type"] == "Ref"
            ]

            if len(self.markers):
                # Set the pos to be the center of the first detected marker
                marker_pos = self.markers[0]["img_pos"]
                self.pos = self.markers[0]["norm_pos"]
            else:
                self.pos = None  # indicate that no reference is detected

            # Check if there are more than one markers
            if len(self.markers) > 1:
                audio.tink()
                logger.warning(
                    "{} markers detected. Please remove all the other markers".
                    format(len(self.markers)))

            # only save a valid ref position if within sample window of calibration routine
            on_position = (self.lead_in < self.screen_marker_state <
                           (self.lead_in + self.sample_duration))

            if on_position and len(self.markers):
                ref = {}
                ref["norm_pos"] = self.pos
                ref["screen_pos"] = marker_pos
                ref["timestamp"] = frame.timestamp
                self.ref_list.append(ref)

            # Always save pupil positions
            self.pupil_list.extend(events["pupil"])

            if on_position and len(self.markers) and events.get(
                    "fixations", []):
                fixation_boost = 5
                self.screen_marker_state = min(
                    self.sample_duration + self.lead_in,
                    self.screen_marker_state + fixation_boost,
                )

            # Animate the screen marker
            if (self.screen_marker_state <
                    self.sample_duration + self.lead_in + self.lead_out):
                if len(self.markers) or not on_position:
                    self.screen_marker_state += 1
            else:
                self.screen_marker_state = 0
                if not self.sites:
                    self.stop()
                    return
                self.active_site = self.sites.pop(0)
                logger.debug("Moving screen marker to site at {} {}".format(
                    *self.active_site))

            # use np.arrays for per element wise math
            self.display_pos = np.array(self.active_site)
            self.on_position = on_position
            self.button.status_text = "{}".format(self.active_site)

        if self._window:
            self.gl_display_in_window()
Esempio n. 9
0
    def recent_events(self, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        frame = events.get('frame')
        if self.active and frame:
            recent_pupil_positions = events['pupil_positions']

            gray_img = frame.gray

            # Update the marker
            self.markers = self.circle_tracker.update(gray_img)

            self.stop_marker_found = False
            if len(self.markers):
                # Set the pos to be the center of the first detected marker
                marker_pos = self.markers[0]['img_pos']
                self.pos = self.markers[0]['norm_pos']
                # Check if there are stop markers
                for marker in self.markers:
                    if marker['marker_type'] == 'Stop':
                        self.auto_stop += 1
                        self.stop_marker_found = True
                        break
            else:
                self.pos = None  # indicate that no reference is detected

            if self.stop_marker_found is False:
                self.auto_stop = 0

            # Check if there are more than one markers
            if len(self.markers) > 1:
                audio.tink()
                logger.warning(
                    "{} markers detected. Please remove all the other markers".
                    format(len(self.markers)))

            # tracking logic
            if len(self.markers) and not self.stop_marker_found:
                # start counter if ref is resting in place and not at last sample site
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother * (
                    new_vel - self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos - np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0]) + abs(
                    sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if self.counter <= 0:
                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug(
                            "Steady marker found. Starting to sample {} datapoints"
                            .format(self.counter_max))
                        self.notify_all({
                            'subject':
                            'calibration.marker_found',
                            'timestamp':
                            self.g_pool.get_timestamp(),
                            'record':
                            True
                        })
                        self.counter = self.counter_max

                if self.counter > 0:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.warning(
                            "Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again."
                            .format(self.counter_max - self.counter))
                        self.notify_all({
                            'subject':
                            'calibration.marker_moved_too_quickly',
                            'timestamp':
                            self.g_pool.get_timestamp(),
                            'record':
                            True
                        })
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = marker_pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if events.get('fixations', []):
                            self.counter -= 5
                        if self.counter <= 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug(
                                "Sampled {} datapoints. Stopping to sample. Looking for steady marker again."
                                .format(self.counter_max))
                            self.notify_all({
                                'subject':
                                'calibration.marker_sample_completed',
                                'timestamp':
                                self.g_pool.get_timestamp(),
                                'record':
                                True
                            })

            # Always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if self.counter:
                if len(self.markers):
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'

            # Stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()

        else:
            pass
Esempio n. 10
0
    def update(self, frame, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        if self.active:
            recent_pupil_positions = events['pupil_positions']

            gray_img = frame.gray

            if self.world_size is None:
                self.world_size = frame.width, frame.height

            self.candidate_ellipses = get_candidate_ellipses(
                gray_img,
                area_threshold=self.area_threshold,
                dist_threshold=self.dist_threshold,
                min_ring_count=5,
                visual_debug=self.show_edges)

            if len(self.candidate_ellipses) > 0:
                self.detected = True
                marker_pos = self.candidate_ellipses[0][0]
                self.pos = normalize(marker_pos, (frame.width, frame.height),
                                     flip_y=True)

            else:
                self.detected = False
                self.pos = None  #indicate that no reference is detected

            # center dark or white?
            if self.detected:
                second_ellipse = self.candidate_ellipses[1]
                col_slice = int(second_ellipse[0][0] - second_ellipse[1][0] /
                                2), int(second_ellipse[0][0] +
                                        second_ellipse[1][0] / 2)
                row_slice = int(second_ellipse[0][1] - second_ellipse[1][1] /
                                2), int(second_ellipse[0][1] +
                                        second_ellipse[1][1] / 2)
                marker_gray = gray_img[slice(*row_slice), slice(*col_slice)]
                avg = cv2.mean(marker_gray)[0]  #CV2 fn return has changed!
                center = marker_gray[second_ellipse[1][1] / 2,
                                     second_ellipse[1][0] / 2]
                rel_shade = center - avg

                #auto_stop logic
                if rel_shade > 30:
                    #bright marker center found
                    self.auto_stop += 1
                    self.stop_marker_found = True

                else:
                    self.auto_stop = 0
                    self.stop_marker_found = False

            #tracking logic
            if self.detected and not self.stop_marker_found:
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother * (
                    new_vel - self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos - np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0]) + abs(
                    sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if not self.counter:

                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug(
                            "Steady marker found. Starting to sample %s datapoints"
                            % self.counter_max)
                        self.counter = self.counter_max

                if self.counter:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.debug(
                            "Marker moved to quickly: Aborted sample. Sampled %s datapoints. Looking for steady marker again."
                            % (self.counter_max - self.counter))
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if self.counter == 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug(
                                "Sampled %s datapoints. Stopping to sample. Looking for steady marker again."
                                % self.counter_max)

            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.g_pool.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if self.counter:
                if self.detected:
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'

            #stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()

        else:
            pass
Esempio n. 11
0
    def recent_events(self, events):
        frame = events.get('frame')
        if not frame:
            return
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        if self.active:
            recent_pupil_positions = events['pupil_positions']

            gray_img = frame.gray

            if self.world_size is None:
                self.world_size = frame.width, frame.height

            self.markers = find_concetric_circles(gray_img, min_ring_count=3)

            if len(self.markers) > 0:
                self.detected = True
                marker_pos = self.markers[0][0][
                    0]  #first marker, innermost ellipse, center
                self.pos = normalize(marker_pos, (frame.width, frame.height),
                                     flip_y=True)

            else:
                self.detected = False
                self.pos = None  #indicate that no reference is detected

            #tracking logic
            if self.detected:
                self.auto_stop += 1
                self.stop_marker_found = True

                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother * (
                    new_vel - self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos - np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0]) + abs(
                    sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if not self.counter:

                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug(
                            "Steady marker found. Starting to sample {} datapoints"
                            .format(self.counter_max))
                        self.counter = self.counter_max

                if self.counter:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.warning(
                            "Marker moved to quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again."
                            .format((self.counter_max - self.counter)))
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = denormalize(
                            self.pos, (frame.width, frame.height), flip_y=True)
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if self.counter == 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug(
                                "Sampled {} datapoints. Stopping to sample. Looking for steady marker again."
                                .format(self.counter_max))

            #always save pupil positions
            for pt in events.get('gaze_positions', []):
                if pt['confidence'] > self.pupil_confidence_threshold:
                    #we add an id for the calibration preprocess data to work as is usually expects pupil data.
                    pt['id'] = 0
                    self.gaze_list.append(pt)

            if self.counter:
                if self.detected:
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'

            # stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()
        else:
            pass
Esempio n. 12
0
    def update(self,frame,recent_pupil_positions,events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        if self.active:
            img = frame.img
            gray_img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
            # self.candidate_points = self.detector.detect(s_img)

            # get threshold image used to get crisp-clean edges
            edges = cv2.adaptiveThreshold(gray_img, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, self.aperture.value, 7)
            # cv2.flip(edges,1 ,dst = edges,)
            # display the image for debugging purpuses
            # img[:] = cv2.cvtColor(edges,cv2.COLOR_GRAY2BGR)
             # from edges to contours to ellipses CV_RETR_CCsOMP ls fr hole
            contours, hierarchy = cv2.findContours(edges,
                                            mode=cv2.RETR_TREE,
                                            method=cv2.CHAIN_APPROX_NONE,offset=(0,0)) #TC89_KCOS


            # remove extra encapsulation
            hierarchy = hierarchy[0]
            # turn outmost list into array
            contours =  np.array(contours)
            # keep only contours                        with parents     and      children
            contained_contours = contours[np.logical_and(hierarchy[:,3]>=0, hierarchy[:,2]>=0)]
            # turn on to debug contours
            if self.show_edges.value:
                cv2.drawContours(img, contained_contours,-1, (0,0,255))

            # need at least 5 points to fit ellipse
            contained_contours =  [c for c in contained_contours if len(c) >= 5]

            ellipses = [cv2.fitEllipse(c) for c in contained_contours]
            self.candidate_ellipses = []
            # filter for ellipses that have similar area as the source contour
            for e,c in zip(ellipses,contained_contours):
                a,b = e[1][0]/2.,e[1][1]/2.
                if abs(cv2.contourArea(c)-np.pi*a*b) <self.area_threshold.value:
                    self.candidate_ellipses.append(e)


            def man_dist(e,other):
                return abs(e[0][0]-other[0][0])+abs(e[0][1]-other[0][1])

            def get_cluster(ellipses):
                for e in ellipses:
                    close_ones = []
                    for other in ellipses:
                        if man_dist(e,other)<self.dist_threshold.value:
                            close_ones.append(other)
                    if len(close_ones)>=3:
                        # sort by major axis to return smallest ellipse first
                        close_ones.sort(key=lambda e: max(e[1]))
                        return close_ones
                return []

            self.candidate_ellipses = get_cluster(self.candidate_ellipses)


            if len(self.candidate_ellipses) > 0:
                self.detected= True
                marker_pos = self.candidate_ellipses[0][0]
                self.pos = normalize(marker_pos,(img.shape[1],img.shape[0]),flip_y=True)

            else:
                self.detected = False
                self.pos = None #indicate that no reference is detected


            if self.detected:
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother*(pos-smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0])+abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother*(new_vel-self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos-np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0])+abs(sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if not self.counter:
                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.2:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug("Steady marker found. Starting to sample %s datapoints" %self.counter_max)
                        self.counter = self.counter_max

            if self.counter and self.detected:
                self.counter -= 1
                ref = {}
                ref["norm_pos"] = self.pos
                ref["timestamp"] = frame.timestamp
                self.ref_list.append(ref)
                if self.counter == 0:
                    #last sample before counter done and moving on
                    audio.tink()
                    logger.debug("Sampled %s datapoints. Stopping to sample. Looking for steady marker again."%self.counter_max)

            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['norm_pupil'] is not None:
                    self.pupil_list.append(p_pt)

            if self.world_size is None:
                self.world_size = img.shape[1],img.shape[0]

        else:
            pass
    def recent_events(self, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        frame = events.get("frame")
        if self.active and frame:
            gray_img = frame.gray
            time_frame = self.g_pool.get_timestamp()

            # Update the marker
            self.markers = self.circle_tracker.update(gray_img)

            self.stop_marker_found = False
            if len(self.markers):
                # Set the pos to be the center of the first detected marker
                marker_pos = self.markers[0]["img_pos"]
                e = self.markers[0]["ellipses"]
                self.pos = self.markers[0]["norm_pos"]
                with open(self.ts_filename, 'a+') as self.ts_file:
                    self.ts_file.write(str(time_frame) + ",")
                    #self.ts_file.write(str(marker_pos[0]) + ',' + str(marker_pos[1]) + "\n")
                    self.ts_file.write(
                        str(e[0][0][0]) + ',' + str(e[0][0][1]) + ',' +
                        str(e[1][1][0]) + ',' + str(e[1][1][1]) + ',' +
                        str(e[0][2]) + "\n")
                    #self.ts_file.write(str(e[0]) + "\n")

                # Check if there are stop markers
                for marker in self.markers:
                    if marker["marker_type"] == "Stop":
                        self.auto_stop += 1
                        self.stop_marker_found = True
                        break
            else:
                self.pos = None  # indicate that no reference is detected

            if self.stop_marker_found is False:
                self.auto_stop = 0

            # Check if there are more than one markers
            if len(self.markers) > 1:
                audio.tink()
                logger.warning(
                    "{} markers detected. Please remove all the other markers".
                    format(len(self.markers)))

            # tracking logic
            if len(self.markers) and not self.stop_marker_found:
                # start counter if ref is resting in place and not at last sample site
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                # manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother * (
                    new_vel - self.smooth_vel)

                # distance to last sampled site
                sample_ref_dist = smooth_pos - np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0]) + abs(
                    sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if self.counter <= 0:
                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug(
                            "Steady marker found. Starting to sample {} datapoints"
                            .format(self.counter_max))
                        self.notify_all({
                            "subject":
                            "calibration.marker_found",
                            "timestamp":
                            self.g_pool.get_timestamp(),
                            "record":
                            True,
                        })
                        self.counter = self.counter_max

                if self.counter > 0:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.warning(
                            "Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again."
                            .format(self.counter_max - self.counter))
                        self.notify_all({
                            "subject":
                            "calibration.marker_moved_too_quickly",
                            "timestamp":
                            self.g_pool.get_timestamp(),
                            "record":
                            True,
                        })
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = marker_pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if self.counter <= 0:
                            # last sample before counter done and moving on
                            audio.tink()
                            logger.debug(
                                "Sampled {} datapoints. Stopping to sample. Looking for steady marker again."
                                .format(self.counter_max))
                            self.notify_all({
                                "subject":
                                "calibration.marker_sample_completed",
                                "timestamp":
                                self.g_pool.get_timestamp(),
                                "record":
                                True,
                            })

            # Always save pupil positions
            self.pupil_list.extend(events["pupil"])

            if self.counter:
                if len(self.markers):
                    self.button.status_text = "Sampling Gaze Data"
                else:
                    self.button.status_text = "Marker Lost"
            else:
                self.button.status_text = "Looking for Marker"

            # Stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()

        else:
            pass
    def recent_events(self, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        frame = events.get('frame')
        if self.active and frame:
            recent_pupil_positions = events['pupil_positions']

            gray_img = frame.gray

            if self.world_size is None:
                self.world_size = frame.width, frame.height

            self.markers = find_concetric_circles(gray_img, min_ring_count=3)

            if len(self.markers) > 0:
                self.detected = True
                marker_pos = self.markers[0][0][
                    0]  #first marker innermost ellipse, pos
                self.pos = normalize(marker_pos, (frame.width, frame.height),
                                     flip_y=True)

            else:
                self.detected = False
                self.pos = None  #indicate that no reference is detected

            # center dark or white?
            if self.detected:
                second_ellipse = self.markers[0][1]
                col_slice = int(second_ellipse[0][0] - second_ellipse[1][0] /
                                2), int(second_ellipse[0][0] +
                                        second_ellipse[1][0] / 2)
                row_slice = int(second_ellipse[0][1] - second_ellipse[1][1] /
                                2), int(second_ellipse[0][1] +
                                        second_ellipse[1][1] / 2)
                marker_gray = gray_img[slice(*row_slice), slice(*col_slice)]
                avg = cv2.mean(marker_gray)[0]
                center = marker_gray[int(second_ellipse[1][1]) // 2,
                                     int(second_ellipse[1][0]) // 2]
                rel_shade = center - avg

                #auto_stop logic
                if rel_shade > 30:
                    #bright marker center found
                    self.auto_stop += 1
                    self.stop_marker_found = True

                else:
                    self.auto_stop = 0
                    self.stop_marker_found = False

            #tracking logic
            if self.detected and not self.stop_marker_found:
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother * (
                    new_vel - self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos - np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0]) + abs(
                    sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if self.counter <= 0:

                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug(
                            "Steady marker found. Starting to sample {} datapoints"
                            .format(self.counter_max))
                        self.notify_all({
                            'subject':
                            'calibration.marker_found',
                            'timestamp':
                            self.g_pool.get_timestamp(),
                            'record':
                            True
                        })
                        self.counter = self.counter_max

                if self.counter > 0:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.warning(
                            "Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again."
                            .format(self.counter_max - self.counter))
                        self.notify_all({
                            'subject':
                            'calibration.marker_moved_too_quickly',
                            'timestamp':
                            self.g_pool.get_timestamp(),
                            'record':
                            True
                        })
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = marker_pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if events.get('fixations', []):
                            self.counter -= self.fixation_boost
                        if self.counter <= 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug(
                                "Sampled {} datapoints. Stopping to sample. Looking for steady marker again."
                                .format(self.counter_max))
                            self.notify_all({
                                'subject':
                                'calibration.marker_sample_completed',
                                'timestamp':
                                self.g_pool.get_timestamp(),
                                'record':
                                True
                            })

            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if self.counter:
                if self.detected:
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'

            #stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()

        else:
            pass
Esempio n. 15
0
    def recent_events(self, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        frame = events.get('frame')

        if frame:
            self.world_size = frame.width, frame.height

            # Check if user selected a new color as threshold
            if self.clicked_color_point is not None:
                pnt = [
                    int(self.clicked_color_point[0][0]),
                    int(self.clicked_color_point[0][1])
                ]

                bgr_color = frame.img[pnt[1]][pnt[0]]
                extracted_color = colorsys.rgb_to_hsv(bgr_color[2] / 255,
                                                      bgr_color[1] / 255,
                                                      bgr_color[0] / 255)
                self.color_h = int(extracted_color[0] * 360)
                self.color_s = int(extracted_color[1] * 100)
                self.color_v = int(extracted_color[2] * 100)

                self.clicked_color_point = None

        if self.active and frame:
            recent_pupil_positions = events['pupil_positions']

            # Normalize HSV color specs to OpenCV HSV color specs
            color_threshold = [
                self.color_h / 2, self.color_s * 2.55, self.color_v * 2.55,
                self.color_tolerance_h / 2, self.color_tolerance_s * 2.55,
                self.color_tolerance_v * 2.55
            ]

            # Detect fingertips
            res = Finger_Detection.findFingers(frame.img, 30, color_threshold,
                                               self.correct_finger_scale)
            fingers = res[0]
            self.contour = res[1]

            # Only update finger positions if we aren't currently collecting data points (if static fingers enabled)
            if self.counter <= 0 or not self.static_finger:
                self.markers = fingers

            # Detected single fingertip
            if len(self.markers) == 1:
                self.detected = True
                marker_pos = [
                    float(self.markers[0][0]),
                    float(self.markers[0][1])
                ]

                self.pos = normalize(marker_pos, (frame.width, frame.height),
                                     flip_y=True)
            else:
                self.detected = False
                self.pos = None  # indicate that no reference is detected

            # Tracking logic
            # Code copied from Pupil's manual marker plugin implementation
            if self.detected:
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother * (
                    new_vel - self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos - np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0]) + abs(
                    sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if self.counter <= 0:
                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        self.end_time = time.time()
                        if self.first_sample:
                            self.first_sample = False
                            self.start_time = time.time()

                        logger.debug(
                            "Steady marker found. Starting to sample {} datapoints"
                            .format(self.counter_max))
                        self.notify_all({
                            'subject':
                            'calibration.marker_found',
                            'timestamp':
                            self.g_pool.get_timestamp(),
                            'record':
                            True
                        })
                        self.counter = self.counter_max
                        self.finger_log.append(self.markers)

                if self.counter > 0:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        self.end_time = time.time()
                        logger.warning(
                            "Marker moved too quickly: Sampled {} datapoints. Looking for steady marker again."
                            .format(self.counter_max - self.counter))
                        self.notify_all({
                            'subject':
                            'calibration.marker_moved_too_quickly',
                            'timestamp':
                            self.g_pool.get_timestamp(),
                            'record':
                            True
                        })
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = marker_pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)

                        if events.get('fixations', []):
                            self.counter -= 5
                        if self.counter <= 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            self.end_time = time.time()
                            logger.info(
                                "Sampled {} datapoints. Stopping to sample. Looking for steady marker again."
                                .format(self.counter_max))
                            self.notify_all({
                                'subject':
                                'calibration.marker_sample_completed',
                                'timestamp':
                                self.g_pool.get_timestamp(),
                                'record':
                                True
                            })

            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if self.counter <= 0:
                self.button.status_text = 'Looking for Marker'
            elif self.counter > 0:
                self.button.status_text = 'Sampling Data'
        else:
            pass
    def recent_events(self, events):
        frame = events.get("frame")
        if self.active and frame:
            gray_img = frame.gray

            if self.clicks_to_close <= 0:
                self.stop()
                return

            # Update the marker
            self.markers = self.circle_tracker.update(gray_img)
            # Screen marker takes only Ref marker
            self.markers = [
                marker for marker in self.markers if marker["marker_type"] == "Ref"
            ]

            if len(self.markers):
                # Set the pos to be the center of the first detected marker
                marker_pos = self.markers[0]["img_pos"]
                self.pos = self.markers[0]["norm_pos"]
            else:
                self.pos = None  # indicate that no reference is detected

            # Check if there are more than one markers
            if len(self.markers) > 1:
                audio.tink()
                logger.warning(
                    "{} markers detected. Please remove all the other markers".format(
                        len(self.markers)
                    )
                )

            # only save a valid ref position if within sample window of calibration routine
            on_position = (
                self.lead_in
                < self.screen_marker_state
                < (self.lead_in + self.sample_duration)
            )

            if on_position and len(self.markers):
                ref = {}
                ref["norm_pos"] = self.pos
                ref["screen_pos"] = marker_pos
                ref["timestamp"] = frame.timestamp
                self.ref_list.append(ref)

            # Always save pupil positions
            self.pupil_list.extend(events["pupil"])

            if on_position and len(self.markers) and events.get("fixations", []):
                fixation_boost = 5
                self.screen_marker_state = min(
                    self.sample_duration + self.lead_in,
                    self.screen_marker_state + fixation_boost,
                )

            # Animate the screen marker
            if (
                self.screen_marker_state
                < self.sample_duration + self.lead_in + self.lead_out
            ):
                if len(self.markers) or not on_position:
                    self.screen_marker_state += 1
            else:
                self.screen_marker_state = 0
                if not self.sites:
                    self.stop()
                    return
                self.active_site = self.sites.pop(0)
                logger.debug(
                    "Moving screen marker to site at {} {}".format(*self.active_site)
                )

            # use np.arrays for per element wise math
            self.display_pos = np.array(self.active_site)
            self.on_position = on_position
            self.button.status_text = "{}".format(self.active_site)

        if self._window:
            self.gl_display_in_window()
Esempio n. 17
0
    def recent_events(self, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        frame = events.get('frame')
        if self.active and frame:
            recent_pupil_positions = events['pupil_positions']

            gray_img  = frame.gray

            if self.world_size is None:
                self.world_size = frame.width,frame.height

            self.markers = find_concetric_circles(gray_img,min_ring_count=3)

            if len(self.markers) > 0:
                self.detected = True
                marker_pos = self.markers[0][0][0] #first marker innermost ellipse, pos
                self.pos = normalize(marker_pos,(frame.width,frame.height),flip_y=True)

            else:
                self.detected = False
                self.pos = None  # indicate that no reference is detected

            # center dark or white?
            if self.detected:
                second_ellipse = self.markers[0][1]
                col_slice = int(second_ellipse[0][0]-second_ellipse[1][0]/2),int(second_ellipse[0][0]+second_ellipse[1][0]/2)
                row_slice = int(second_ellipse[0][1]-second_ellipse[1][1]/2),int(second_ellipse[0][1]+second_ellipse[1][1]/2)
                marker_gray = gray_img[slice(*row_slice),slice(*col_slice)]
                avg = cv2.mean(marker_gray)[0]
                center = marker_gray[int(second_ellipse[1][1])//2, int(second_ellipse[1][0])//2]
                rel_shade = center-avg

                # auto_stop logic
                if rel_shade > 30:
                    # bright marker center found
                    self.auto_stop +=1
                    self.stop_marker_found = True

                else:
                    self.auto_stop = 0
                    self.stop_marker_found = False


            #tracking logic
            if self.detected and not self.stop_marker_found:
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother*(pos-smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                #manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0])+abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother*(new_vel-self.smooth_vel)

                #distance to last sampled site
                sample_ref_dist = smooth_pos-np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0])+abs(sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if self.counter <= 0:

                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug("Steady marker found. Starting to sample {} datapoints".format(self.counter_max))
                        self.notify_all({'subject':'calibration.marker_found','timestamp':self.g_pool.get_timestamp(),'record':True})
                        self.counter = self.counter_max

                if self.counter > 0:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.warning("Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again.".format(self.counter_max-self.counter))
                        self.notify_all({'subject':'calibration.marker_moved_too_quickly','timestamp':self.g_pool.get_timestamp(),'record':True})
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = marker_pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if events.get('fixations', []):
                            self.counter -= self.fixation_boost
                        if self.counter <= 0:
                            #last sample before counter done and moving on
                            audio.tink()
                            logger.debug("Sampled {} datapoints. Stopping to sample. Looking for steady marker again.".format(self.counter_max))
                            self.notify_all({'subject':'calibration.marker_sample_completed','timestamp':self.g_pool.get_timestamp(),'record':True})


            #always save pupil positions
            for p_pt in recent_pupil_positions:
                if p_pt['confidence'] > self.pupil_confidence_threshold:
                    self.pupil_list.append(p_pt)

            if self.counter:
                if self.detected:
                    self.button.status_text = 'Sampling Gaze Data'
                else:
                    self.button.status_text = 'Marker Lost'
            else:
                self.button.status_text = 'Looking for Marker'



            #stop if autostop condition is satisfied:
            if self.auto_stop >=self.auto_stop_max:
                self.auto_stop = 0
                self.stop()


        else:
            pass
    def recent_events(self, events):
        """
        gets called once every frame.
        reference positon need to be published to shared_pos
        if no reference was found, publish 0,0
        """
        frame = events.get("frame")
        if self.active and frame:
            gray_img = frame.gray

            # Update the marker
            self.markers = self.circle_tracker.update(gray_img)

            self.stop_marker_found = False
            if len(self.markers):
                # Set the pos to be the center of the first detected marker
                marker_pos = self.markers[0]["img_pos"]
                self.pos = self.markers[0]["norm_pos"]
                # Check if there are stop markers
                for marker in self.markers:
                    if marker["marker_type"] == "Stop":
                        self.auto_stop += 1
                        self.stop_marker_found = True
                        break
            else:
                self.pos = None  # indicate that no reference is detected

            if self.stop_marker_found is False:
                self.auto_stop = 0

            # Check if there are more than one markers
            if len(self.markers) > 1:
                audio.tink()
                logger.warning(
                    "{} markers detected. Please remove all the other markers".format(
                        len(self.markers)
                    )
                )

            # tracking logic
            if len(self.markers) and not self.stop_marker_found:
                # start counter if ref is resting in place and not at last sample site
                # calculate smoothed manhattan velocity
                smoother = 0.3
                smooth_pos = np.array(self.smooth_pos)
                pos = np.array(self.pos)
                new_smooth_pos = smooth_pos + smoother * (pos - smooth_pos)
                smooth_vel_vec = new_smooth_pos - smooth_pos
                smooth_pos = new_smooth_pos
                self.smooth_pos = list(smooth_pos)
                # manhattan distance for velocity
                new_vel = abs(smooth_vel_vec[0]) + abs(smooth_vel_vec[1])
                self.smooth_vel = self.smooth_vel + smoother * (
                    new_vel - self.smooth_vel
                )

                # distance to last sampled site
                sample_ref_dist = smooth_pos - np.array(self.sample_site)
                sample_ref_dist = abs(sample_ref_dist[0]) + abs(sample_ref_dist[1])

                # start counter if ref is resting in place and not at last sample site
                if self.counter <= 0:
                    if self.smooth_vel < 0.01 and sample_ref_dist > 0.1:
                        self.sample_site = self.smooth_pos
                        audio.beep()
                        logger.debug(
                            "Steady marker found. Starting to sample {} datapoints".format(
                                self.counter_max
                            )
                        )
                        self.notify_all(
                            {
                                "subject": "calibration.marker_found",
                                "timestamp": self.g_pool.get_timestamp(),
                                "record": True,
                            }
                        )
                        self.counter = self.counter_max

                if self.counter > 0:
                    if self.smooth_vel > 0.01:
                        audio.tink()
                        logger.warning(
                            "Marker moved too quickly: Aborted sample. Sampled {} datapoints. Looking for steady marker again.".format(
                                self.counter_max - self.counter
                            )
                        )
                        self.notify_all(
                            {
                                "subject": "calibration.marker_moved_too_quickly",
                                "timestamp": self.g_pool.get_timestamp(),
                                "record": True,
                            }
                        )
                        self.counter = 0
                    else:
                        self.counter -= 1
                        ref = {}
                        ref["norm_pos"] = self.pos
                        ref["screen_pos"] = marker_pos
                        ref["timestamp"] = frame.timestamp
                        self.ref_list.append(ref)
                        if events.get("fixations", []):
                            self.counter -= 5
                        if self.counter <= 0:
                            # last sample before counter done and moving on
                            audio.tink()
                            logger.debug(
                                "Sampled {} datapoints. Stopping to sample. Looking for steady marker again.".format(
                                    self.counter_max
                                )
                            )
                            self.notify_all(
                                {
                                    "subject": "calibration.marker_sample_completed",
                                    "timestamp": self.g_pool.get_timestamp(),
                                    "record": True,
                                }
                            )

            # Always save pupil positions
            self.pupil_list.extend(events["pupil"])

            if self.counter:
                if len(self.markers):
                    self.button.status_text = "Sampling Gaze Data"
                else:
                    self.button.status_text = "Marker Lost"
            else:
                self.button.status_text = "Looking for Marker"

            # Stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()

        else:
            pass
    def recent_events(self, events):
        frame = events.get("frame")
        if self.active and frame:
            gray_img = frame.gray

            if self.clicks_to_close <= 0:
                self.stop()
                return

            # Update the marker
            self.markers = self.circle_tracker.update(gray_img)

            self.stop_marker_found = False
            if len(self.markers):
                # Set the pos to be the center of the first detected marker
                marker_pos = self.markers[0]["img_pos"]
                self.pos = self.markers[0]["norm_pos"]
                # Check if there are stop markers
                for marker in self.markers:
                    if marker["marker_type"] == "Stop":
                        self.auto_stop += 1
                        self.stop_marker_found = True
                        break
            else:
                self.pos = None  # indicate that no reference is detected

            if self.stop_marker_found is False:
                self.auto_stop = 0

            # Check if there are more than one markers
            if len(self.markers) > 1:
                audio.tink()
                logger.warning(
                    "{} markers detected. Please remove all the other markers".format(
                        len(self.markers)
                    )
                )

            # only save a valid ref position if within sample window of calibraiton routine
            on_position = self.lead_in < self.screen_marker_state

            if on_position and len(self.markers) and not self.stop_marker_found:
                ref = {}
                ref["norm_pos"] = self.pos
                ref["screen_pos"] = marker_pos
                ref["timestamp"] = frame.timestamp
                self.ref_list.append(ref)

            # always save pupil positions
            self.pupil_list.extend(events["pupil"])

            # Animate the screen marker
            if len(self.markers) or not on_position:
                self.screen_marker_state += 1

            # Stop if autostop condition is satisfied:
            if self.auto_stop >= self.auto_stop_max:
                self.auto_stop = 0
                self.stop()

            # use np.arrays for per element wise math
            self.on_position = on_position
        if self._window:
            self.gl_display_in_window()