Exemple #1
0
 def _start_priority_queue(self):
     if (self._update_prio_queue_thread is None and (self.delay_after_timestamp_prio is not None or self.mode ==
                                                     "iv_mitm") and not self.mode == "pokestops"):
         self._prio_queue = []
         if self.mode not in ["iv_mitm", "pokestops"]:
             self.clustering_helper = ClusteringHelper(self._max_radius,
                                                       self._max_coords_within_radius,
                                                       self._cluster_priority_queue_criteria())
         self._update_prio_queue_thread = Thread(name="prio_queue_update_" + self.name,
                                                 target=self._update_priority_queue_loop)
         self._update_prio_queue_thread.daemon = False
         self._update_prio_queue_thread.start()
Exemple #2
0
    def __init__(self, db_wrapper, coords, max_radius, max_coords_within_radius, path_to_include_geofence,
                 path_to_exclude_geofence, routefile, mode=None, init=False,
                 name="unknown", settings=None):
        self.db_wrapper = db_wrapper
        self.init = init
        self.name = name
        self._coords_unstructured = coords
        self.geofence_helper = GeofenceHelper(
            path_to_include_geofence, path_to_exclude_geofence)
        self._routefile = routefile
        self._max_radius = max_radius
        self._max_coords_within_radius = max_coords_within_radius
        self.settings = settings
        self.mode = mode

        self._last_round_prio = False
        self._manager_mutex = Lock()
        self._round_started_time = None
        if coords is not None:
            if init:
                fenced_coords = coords
            else:
                fenced_coords = self.geofence_helper.get_geofenced_coordinates(
                    coords)
            self._route = getJsonRoute(
                fenced_coords, max_radius, max_coords_within_radius, routefile)
        else:
            self._route = None
        self._current_index_of_route = 0
        self._init_mode_rounds = 0
        if settings is not None:
            self.delay_after_timestamp_prio = settings.get(
                "delay_after_prio_event", None)
            if self.delay_after_timestamp_prio == 0:
                self.delay_after_timestamp_prio = None
            self.starve_route = settings.get("starve_route", False)
        else:
            self.delay_after_timestamp_prio = None
            self.starve_route = False
        if self.delay_after_timestamp_prio is not None or mode == "iv_mitm":
            self._prio_queue = []
            if mode != "iv_mitm":
                self.clustering_helper = ClusteringHelper(self._max_radius,
                                                          self._max_coords_within_radius,
                                                          self._cluster_priority_queue_criteria())
            self._stop_update_thread = Event()
            self._update_prio_queue_thread = Thread(name="prio_queue_update_" + name,
                                                    target=self._update_priority_queue_loop)
            self._update_prio_queue_thread.daemon = True
            self._update_prio_queue_thread.start()
        else:
            self._prio_queue = None
Exemple #3
0
    def getLessCoords(self, npCoordinates, maxRadius, maxCountPerCircle, useS2: bool=False, S2level: int=15):
        coordinates = []
        for coord in npCoordinates:
            coordinates.append(
                (0, Location(coord[0].item(), coord[1].item()))
            )

        clustering_helper = ClusteringHelper(max_radius=maxRadius, max_count_per_circle=maxCountPerCircle,
                                             max_timedelta_seconds=0, useS2=useS2, S2level=S2level)
        clustered_events = clustering_helper.get_clustered(coordinates)
        coords_cleaned_up = []
        for event in clustered_events:
            coords_cleaned_up.append(event[1])
        return coords_cleaned_up
Exemple #4
0
 def _start_priority_queue(self):
     logger.info("Try to activate PrioQ thread for route {}".format(str(self.name)))
     if (self.delay_after_timestamp_prio is not None or self.mode == "iv_mitm") and not self.mode == "pokestops":
         logger.info("PrioQ thread for route {} could be activate".format(str(self.name)))
         self._prio_queue = []
         if self.mode not in ["iv_mitm", "pokestops"]:
             self.clustering_helper = ClusteringHelper(self._max_radius,
                                                       self._max_coords_within_radius,
                                                       self._cluster_priority_queue_criteria())
         self._update_prio_queue_thread = Thread(name="prio_queue_update_" + self.name,
                                                 target=self._update_priority_queue_loop)
         self._update_prio_queue_thread.daemon = True
         self._update_prio_queue_thread.start()
     else:
         logger.info("Cannot activate Prio Q - maybe wrong mode or delay_after_prio_event is null")
Exemple #5
0
def getLessCoords(npCoordinates, maxRadius, maxCountPerCircle):
    coordinates = []
    for coord in npCoordinates:
        coordinates.append((0, Location(coord[0].item(), coord[1].item())))

    clustering_helper = ClusteringHelper(
        max_radius=maxRadius,
        max_count_per_circle=maxCountPerCircle,
        max_timedelta_seconds=0)
    clustered_events = clustering_helper.get_clustered(coordinates)
    # relations = __getRelationsInRange(coordinates, maxRadius)
    # summedUp = __sumUpRelations(relations, maxCountPerCircle, maxRadius)
    coords_cleaned_up = []
    for event in clustered_events:
        coords_cleaned_up.append(event[1])

    # print "Done summing up: " + str(summedUp) + " that's just " + str(len(summedUp))
    return coords_cleaned_up
Exemple #6
0
class RouteManagerBase(ABC):
    def __init__(self, db_wrapper: DbWrapperBase, dbm: DataManager, area_id: str, coords: List[Location], max_radius: float,
                 max_coords_within_radius: int, path_to_include_geofence: str, path_to_exclude_geofence: str,
                 routefile: str, mode=None, init: bool = False, name: str = "unknown", settings: dict = None,
                 level: bool = False, calctype: str = "optimized", useS2: bool = False, S2level: int = 15, joinqueue = None):
        self.db_wrapper: DbWrapperBase = db_wrapper
        self.init: bool = init
        self.name: str = name
        self._data_manager = dbm
        self.useS2: bool = useS2
        self.S2level: int = S2level
        self.area_id = area_id

        self._coords_unstructured: List[Location] = coords
        self.geofence_helper: GeofenceHelper = GeofenceHelper(
            path_to_include_geofence, path_to_exclude_geofence)
        self._routefile = os.path.join(args.file_path, routefile)
        self._max_radius: float = max_radius
        self._max_coords_within_radius: int = max_coords_within_radius
        self.settings: dict = settings
        self.mode = mode
        self._is_started: bool = False
        self._first_started = False
        self._current_route_round_coords: List[Location] = []
        self._start_calc: bool = False
        self._positiontyp = {}
        self._coords_to_be_ignored = set()
        self._level = level
        self._calctype = calctype
        self._overwrite_calculation: bool = False
        self._stops_not_processed: Dict[Location, int] = {}
        self._routepool: Dict[str, RoutePoolEntry] = {}
        self._roundcount: int = 0
        self._joinqueue = joinqueue

        # we want to store the workers using the routemanager
        self._workers_registered: List[str] = []
        self._workers_registered_mutex = RLock()

        # waiting till routepool is filled up
        self._workers_fillup_mutex = RLock()

        self._last_round_prio = {}
        self._manager_mutex = RLock()
        self._round_started_time = None
        self._route: List[Location] = []

        if coords is not None:
            if init:
                fenced_coords = coords
            else:
                fenced_coords = self.geofence_helper.get_geofenced_coordinates(
                    coords)
            new_coords = getJsonRoute(
                fenced_coords, max_radius, max_coords_within_radius, routefile,
                algorithm=calctype)
            for coord in new_coords:
                self._route.append(Location(coord["lat"], coord["lng"]))
        self._current_index_of_route = 0
        self._init_mode_rounds = 0

        if self.settings is not None:
            self.delay_after_timestamp_prio = self.settings.get(
                "delay_after_prio_event", None)
            self.starve_route = self.settings.get("starve_route", False)
        else:
            self.delay_after_timestamp_prio = None
            self.starve_route = False

        # initialize priority queue variables
        self._prio_queue = None
        self._update_prio_queue_thread = None
        self._check_routepools_thread = None
        self._stop_update_thread = Event()

    def get_ids_iv(self) -> Optional[List[int]]:
        if self.settings is not None:
            return self.settings.get("mon_ids_iv_raw", [])
        else:
            return None

    def _start_check_routepools(self):
        self._check_routepools_thread = Thread(name="_check_routepools_" + self.name,
                                               target=self._check_routepools)
        self._check_routepools_thread.daemon = True
        self._check_routepools_thread.start()

    def join_threads(self):
        logger.info("Shutdown Route Threads")
        if self._update_prio_queue_thread is not None:
            while self._update_prio_queue_thread.isAlive():
                time.sleep(1)
                logger.debug("Shutdown Prio Queue Thread - waiting...")
                self._update_prio_queue_thread.join(5)
        logger.debug("Shutdown Prio Queue Thread - done...")
        if self._check_routepools_thread is not None:
            while self._check_routepools_thread.isAlive():
                time.sleep(1)
                logger.debug("Shutdown Routepool Thread - waiting...")
                self._check_routepools_thread.join(5)

        self._update_prio_queue_thread = None
        self._check_routepools_thread = None
        self._stop_update_thread.clear()
        logger.info("Shutdown Route Threads completed")

    def stop_routemanager(self, joinwithqueue=True):
        # call routetype stoppper
        if self._joinqueue is not None and joinwithqueue:
            logger.info("Adding route {} to queue".format(str(self.name)))
            self._joinqueue.set_queue(self.name)

        self._quit_route()
        self._stop_update_thread.set()

        logger.info("Shutdown of route {} completed".format(str(self.name)))

    def _init_route_queue(self):
        with self._manager_mutex:
            if len(self._route) > 0:
                self._current_route_round_coords.clear()
                logger.debug("Creating queue for coords")
                for latlng in self._route:
                    self._current_route_round_coords.append(latlng)
                logger.debug("Finished creating queue")

    def _clear_coords(self):
        with self._manager_mutex:
            self._coords_unstructured = None

    def register_worker(self, worker_name) -> bool:
        self._workers_registered_mutex.acquire()
        try:
            if worker_name in self._workers_registered:
                logger.info("Worker {} already registered to routemanager {}", str(
                    worker_name), str(self.name))
                return False
            else:
                logger.info("Worker {} registering to routemanager {}",
                            str(worker_name), str(self.name))
                self._workers_registered.append(worker_name)
                self._positiontyp[worker_name] = 0
                return True

        finally:
            self._workers_registered_mutex.release()

    def unregister_worker(self, worker_name):
        self._workers_registered_mutex.acquire()
        try:
            if worker_name in self._workers_registered:
                logger.info("Worker {} unregistering from routemanager {}", str(
                    worker_name), str(self.name))
                self._workers_registered.remove(worker_name)
                if worker_name in self._routepool:
                    logger.info("Deleting old routepool of {}", str(worker_name))
                    del self._routepool[worker_name]
            else:
                # TODO: handle differently?
                logger.info(
                    "Worker {} failed unregistering from routemanager {} since subscription was previously lifted",
                    str(
                        worker_name), str(self.name))
                if worker_name in self._routepool:
                    logger.info("Deleting old routepool of {}", str(worker_name))
                    del self._routepool[worker_name]
            if len(self._workers_registered) == 0 and self._is_started:
                logger.info(
                    "Routemanager {} does not have any subscribing workers anymore, calling stop", str(self.name))
                self.stop_routemanager()
        finally:
            self._workers_registered_mutex.release()

    def stop_worker(self):
        self._workers_registered_mutex.acquire()
        try:
            for worker in self._workers_registered:
                logger.info("Worker {} stopped from routemanager {}", str(
                    worker), str(self.name))
                worker.stop_worker()
                self._workers_registered.remove(worker)
                if worker in self._routepool:
                    logger.info("Deleting old routepool of {}", str(worker))
                    del self._routepool[worker]
            if len(self._workers_registered) == 0 and self._is_started:
                logger.info(
                    "Routemanager {} does not have any subscribing workers anymore, calling stop", str(self.name))
                self.stop_routemanager()
        finally:
            self._workers_registered_mutex.release()

    def _check_started(self):
        return self._is_started

    def _start_priority_queue(self):
        logger.info("Try to activate PrioQ thread for route {}".format(str(self.name)))
        if (self.delay_after_timestamp_prio is not None or self.mode == "iv_mitm") and not self.mode == "pokestops":
            logger.info("PrioQ thread for route {} could be activate".format(str(self.name)))
            self._prio_queue = []
            if self.mode not in ["iv_mitm", "pokestops"]:
                self.clustering_helper = ClusteringHelper(self._max_radius,
                                                          self._max_coords_within_radius,
                                                          self._cluster_priority_queue_criteria())
            self._update_prio_queue_thread = Thread(name="prio_queue_update_" + self.name,
                                                    target=self._update_priority_queue_loop)
            self._update_prio_queue_thread.daemon = True
            self._update_prio_queue_thread.start()
        else:
            logger.info("Cannot activate Prio Q - maybe wrong mode or delay_after_prio_event is null")

    # list_coords is a numpy array of arrays!
    def add_coords_numpy(self, list_coords: np.ndarray):
        fenced_coords = self.geofence_helper.get_geofenced_coordinates(
            list_coords)
        with self._manager_mutex:
            if self._coords_unstructured is None:
                self._coords_unstructured = fenced_coords
            else:
                self._coords_unstructured = np.concatenate(
                    (self._coords_unstructured, fenced_coords))

    def add_coords_list(self, list_coords: List[Location]):
        to_be_appended = np.zeros(shape=(len(list_coords), 2))
        for i in range(len(list_coords)):
            to_be_appended[i][0] = float(list_coords[i].lat)
            to_be_appended[i][1] = float(list_coords[i].lng)
        self.add_coords_numpy(to_be_appended)

    def calculate_new_route(self, coords, max_radius, max_coords_within_radius, routefile, delete_old_route,
                            num_procs=0):
        if self._overwrite_calculation:
            calctype = 'quick'
        else:
            calctype = self._calctype

        if delete_old_route and os.path.exists(str(routefile) + ".calc"):
            logger.debug("Deleting routefile...")
            os.remove(str(routefile) + ".calc")
        new_route = getJsonRoute(coords, max_radius, max_coords_within_radius, num_processes=num_procs,
                                 routefile=routefile, algorithm=calctype, useS2=self.useS2, S2level=self.S2level)
        if self._overwrite_calculation:
            self._overwrite_calculation = False
        return new_route

    def empty_routequeue(self):
        return len(self._current_route_round_coords) > 0

    def recalc_route(self, max_radius: float, max_coords_within_radius: int, num_procs: int = 1,
                     delete_old_route: bool = False, nofile: bool = False):
        current_coords = self._coords_unstructured
        if nofile:
            routefile = None
        else:
            routefile = self._routefile
        new_route = self.calculate_new_route(current_coords, max_radius, max_coords_within_radius,
                                             routefile, delete_old_route, num_procs)
        with self._manager_mutex:
            self._route.clear()
            for coord in new_route:
                self._route.append(Location(coord["lat"], coord["lng"]))
            self._current_route_round_coords = self._route.copy()
            self._current_index_of_route = 0

    def _update_priority_queue_loop(self):
        if self._priority_queue_update_interval() is None or self._priority_queue_update_interval() == 0:
            return
        while not self._stop_update_thread.is_set():
            # retrieve the latest hatches from DB
            # newQueue = self._db_wrapper.get_next_raid_hatches(self._delayAfterHatch, self._geofenceHelper)
            new_queue = self._retrieve_latest_priority_queue()
            self._merge_priority_queue(new_queue)
            redocounter = 0
            while redocounter <= self._priority_queue_update_interval() and not self._stop_update_thread.is_set():
                redocounter += 1
                time.sleep(1)
                if self._stop_update_thread.is_set():
                    logger.info("Kill Prio Queue loop while sleeping")
                    break

    def _merge_priority_queue(self, new_queue):
        if new_queue is not None:
            with self._manager_mutex:
                merged = list(new_queue)
                logger.info("New raw priority queue with {} entries", len(merged))
                merged = self._filter_priority_queue_internal(merged)
                heapq.heapify(merged)
                self._prio_queue = merged
            logger.info("New clustered priority queue with {} entries", len(merged))
            logger.debug("Priority queue entries: {}", str(merged))

    def date_diff_in_seconds(self, dt2, dt1):
        timedelta = dt2 - dt1
        return timedelta.days * 24 * 3600 + timedelta.seconds

    def dhms_from_seconds(self, seconds):
        minutes, seconds = divmod(seconds, 60)
        hours, minutes = divmod(minutes, 60)
        # days, hours = divmod(hours, 24)
        return hours, minutes, seconds

    def _get_round_finished_string(self):
        round_finish_time = datetime.now()
        round_completed_in = (
                "%d hours, %d minutes, %d seconds" % (
            self.dhms_from_seconds(
                self.date_diff_in_seconds(
                    round_finish_time, self._round_started_time)
            )
        )
        )
        return round_completed_in

    def add_coord_to_be_removed(self, lat: float, lon: float):
        if lat < -90.0 or lat > 90.0 or lon < -180.0 or lon > 180.0:
            return
        with self._manager_mutex:
            self._coords_to_be_ignored.add(Location(lat, lon))

    @abstractmethod
    def _retrieve_latest_priority_queue(self):
        """
        Method that's supposed to return a plain list containing (timestamp, Location) of the next events of interest
        :return:
        """
        pass

    @abstractmethod
    def _start_routemanager(self):
        """
        Starts priority queue or whatever the implementations require
        :return:
        """
        pass

    @abstractmethod
    def _quit_route(self):
        """
        Killing the Route Thread
        :return:
        """
        pass

    @abstractmethod
    def _get_coords_post_init(self):
        """
        Return list of coords to be fetched and used for routecalc
        :return:
        """
        pass

    @abstractmethod
    def _check_coords_before_returning(self, lat, lng, origin):
        """
        Return list of coords to be fetched and used for routecalc
        :return:
        """
        pass

    @abstractmethod
    def _recalc_route_workertype(self):
        """
        Return a new route for worker
        :return:
        """
        pass

    @abstractmethod
    def _get_coords_after_finish_route(self) -> bool:
        """
        Return list of coords to be fetched after finish a route
        :return:
        """
        pass

    @abstractmethod
    def _cluster_priority_queue_criteria(self):
        """
        If you do not want to have any filtering, simply return 0, 0, otherwise simply
        return timedelta_seconds, distance
        :return:
        """

    @abstractmethod
    def _priority_queue_update_interval(self):
        """
        The time to sleep in between consecutive updates of the priority queue
        :return:
        """

    @abstractmethod
    def _delete_coord_after_fetch(self) -> bool:
        """
        Whether coords fetched from get_next_location should be removed from the total route
        :return:
        """

    def _filter_priority_queue_internal(self, latest):
        """
        Filter through the internal priority queue and cluster events within the timedelta and distance returned by
        _cluster_priority_queue_criteria
        :return:
        """
        # timedelta_seconds = self._cluster_priority_queue_criteria()
        if self.mode == "iv_mitm":
            # exclude IV prioQ to also pass encounterIDs since we do not pass additional information through when
            # clustering
            return latest
        delete_seconds_passed = 0
        if self.settings is not None:
            delete_seconds_passed = self.settings.get(
                "remove_from_queue_backlog", 0)

        if delete_seconds_passed is not None:
            delete_before = time.time() - delete_seconds_passed
        else:
            delete_before = 0
        latest = [to_keep for to_keep in latest if not to_keep[0] < delete_before]
        # TODO: sort latest by modified flag of event
        # merged = self._merge_queue(latest, self._max_radius, 2, timedelta_seconds)
        merged = self.clustering_helper.get_clustered(latest)
        return merged

    def __set_routepool_entry_location(self, origin: str, pos: Location):
        with self._manager_mutex:
            if self._routepool.get(origin, None) is not None:
                self._routepool[origin].current_pos = pos
                self._routepool[origin].last_access = time.time()
                self._routepool[origin].worker_sleeping = 0

    def get_next_location(self, origin: str) -> Optional[Location]:
        logger.debug("get_next_location of {} called", str(self.name))

        if not self._is_started:
            logger.info(
                "Starting routemanager {} in get_next_location", str(self.name))
            if not self._start_routemanager():
                logger.info('No coords available - quit worker')
                return None

        if self._start_calc:
            logger.info("Another process already calculate the new route")
            return None

        with self._manager_mutex:
            with self._workers_registered_mutex:
                if origin not in self._workers_registered:
                    self.register_worker(origin)

            if origin not in self._routepool:
                logger.debug("No subroute/routepool entry of {} present, creating it", origin)
                self._routepool[origin] = RoutePoolEntry(time.time(), collections.deque(), [],
                                                         time_added=time.time())
                if not self.__worker_changed_update_routepools():
                    logger.info("Failed updating routepools after adding a worker to it")
                    return None

            elif self._routepool[origin].has_prio_event and self.mode != 'iv_mitm':
                prioevent = self._routepool[origin].prio_coords
                logger.info('Worker {} getting a nearby prio event {}', origin, prioevent)
                self._routepool[origin].has_prio_event = False
                self.__set_routepool_entry_location(origin, prioevent)
                self._routepool[origin].last_round_prio_event = True
                return prioevent

        # first check if a location is available, if not, block until we have one...
        got_location = False
        while not got_location and self._is_started and not self.init:
            logger.debug(
                "{}: Checking if a location is available...", str(self.name))
            with self._manager_mutex:
                if self.mode == "iv_mitm":
                    got_location = self._prio_queue is not None and len(self._prio_queue) > 0
                    if not got_location: time.sleep(1)
                else:
                    # normal mode - should always have a route
                    got_location = True

        logger.debug(
            "{}: Location available, acquiring lock and trying to return location", self.name)
        with self._manager_mutex:
            # check priority queue for items of priority that are past our time...
            # if that is not the case, simply increase the index in route and return the location on route

            # determine whether we move to the next location or the prio queue top's item
            if (self.delay_after_timestamp_prio is not None and ((not self._last_round_prio.get(origin, False)
                                                                  or self.starve_route)
                                                                 and self._prio_queue and len(self._prio_queue) > 0
                                                                 and self._prio_queue[0][0] < time.time())):
                logger.debug("{}: Priority event", self.name)
                next_coord = heapq.heappop(self._prio_queue)[1]
                if self._other_worker_closer_to_prioq(next_coord, origin):
                    self._last_round_prio[origin] = True
                    self._positiontyp[origin] = 1
                    return self.get_next_location(origin)
                self._last_round_prio[origin] = True
                self._positiontyp[origin] = 1
                logger.info("Route {} is moving to {}, {} for a priority event",
                            self.name, next_coord.lat, next_coord.lng)
                if self.check_coord_and_maybe_del(next_coord, origin) is None:
                    # Recurse
                    return self.get_next_location(origin)

            logger.debug("{}: Moving on with route", self.name)
            self._positiontyp[origin] = 0
            # TODO: this check is likely always true now.............
            if self.check_worker_rounds() > self._roundcount:
                self._roundcount = self.check_worker_rounds()
                if self._round_started_time is not None:
                    logger.info("All subroutes of {} reached the first spot again. It took {}",
                                self.name, self._get_round_finished_string())
                self._round_started_time = datetime.now()
                if len(self._route) == 0:
                    return None
                logger.info("Round of route {} started at {}",
                            self.name, self._round_started_time)
            elif self._round_started_time is None:
                self._round_started_time = datetime.now()

            if len(self._routepool[origin].queue) == 0:
                # worker do the part of route
                self._routepool[origin].rounds += 1

            # continue as usual
            if self.init and self.check_worker_rounds() >= int(self.settings.get("init_mode_rounds", 1)) and \
                    len(self._routepool[origin].queue) == 0:
                # we are done with init, let's calculate a new route
                logger.warning("Init of {} done, it took {}, calculating new route...",
                               self.name, self._get_round_finished_string())
                if self._start_calc:
                    logger.info("Another process already calculate the new route")
                    return None
                self._start_calc = True
                self._clear_coords()
                coords = self._get_coords_post_init()
                logger.debug("Setting {} coords to as new points in route of {}",
                             len(coords), self.name)
                self.add_coords_list(coords)
                logger.debug("Route of {} is being calculated", self.name)
                self._recalc_route_workertype()
                self.init = False
                self.change_init_mapping(self.name)
                self._start_calc = False
                logger.debug("Initroute of {} is finished - restart worker", self.name)
                return None

            elif len(self._current_route_round_coords) >= 0 and len(self._routepool[origin].queue) == 0:
                # only quest could hit this else!
                logger.info("Worker finished his subroute, updating all subroutes if necessary")

                if self.mode == 'pokestops' and not self.init:
                    # check for coords not in other workers to get a real open coord list
                    if not self._get_coords_after_finish_route():
                        logger.info("No more coords available - dont update routepool")
                        return None

                if not self.__worker_changed_update_routepools():
                    logger.info("Failed updating routepools ...")
                    return None

                if len(self._routepool[origin].queue) == 0 and len(self._routepool[origin].subroute) == 0:
                    logger.info("Subroute-update won't help or queue and subroute are empty, "
                                "signalling worker to reconnect")
                    self._routepool[origin].last_access = time.time()
                    return None
                elif len(self._routepool[origin].queue) == 0 and len(self._routepool[origin].subroute) > 0:
                    [self._routepool[origin].queue.append(i) for i in self._routepool[origin].subroute]
                elif len(self._routepool[origin].queue) > 0 and len(self._routepool[origin].subroute) > 0:
                    logger.info("Getting new coords for route {}", str(self.name))
                else:
                    logger.info("Not getting new coords - leaving worker")
                    return None

            if len(self._routepool[origin].queue) == 0:
                logger.warning("Having updated routepools and checked lengths of queue and subroute, "
                               "{}'s queue is still empty, signalling worker to stop whatever he is doing",
                               self.name)
                self._routepool[origin].last_access = time.time()
                return None

            while len(self._routepool[origin].queue) > 0:
                next_coord = self._routepool[origin].queue.popleft()
                if self._delete_coord_after_fetch() and next_coord in self._current_route_round_coords \
                        and not self.init:
                    self._current_route_round_coords.remove(next_coord)
                logger.info("{}: Moving on with location {} [{} coords left (Workerpool)]",
                            str(self.name), str(next_coord), str(len(self._routepool[origin].queue)))

                self._last_round_prio[origin] = False
                self._routepool[origin].last_round_prio_event = False
                the_coord = self.check_coord_and_maybe_del(next_coord, origin)
                if the_coord is not None:
                    return the_coord

            # No more coords in the queue
            return None

    def check_coord_and_maybe_del(self, next_coord, origin):
        logger.debug("{}: Done grabbing next coord, releasing lock and returning location: {}", str(
            self.name), str(next_coord))
        if self._check_coords_before_returning(next_coord.lat, next_coord.lng, origin):

            if self._delete_coord_after_fetch() and next_coord in self._current_route_round_coords \
                    and not self.init:
                self._current_route_round_coords.remove(next_coord)

            self.__set_routepool_entry_location(origin, next_coord)

            return next_coord
        return None

    def check_worker_rounds(self) -> int:
        temp_worker_round_list: list = []
        with self._manager_mutex:
            for origin, entry in self._routepool.items():
                temp_worker_round_list.append(entry.rounds)

        return 0 if len(temp_worker_round_list) == 0 else min(temp_worker_round_list)

    def _other_worker_closer_to_prioq(self, prioqcoord, origin):
        logger.debug('Check distances from worker to prioQ coord')
        temp_distance: float = 0.0
        closer_worker = None
        if len(self._workers_registered) == 1:
            logger.debug('Route has only one worker - no distance check')
            return False

        current_worker_pos = self._routepool[origin].current_pos
        distance_worker = get_distance_of_two_points_in_meters(current_worker_pos.lat, current_worker_pos.lng,
                                                               prioqcoord.lat, prioqcoord.lng)

        logger.debug("Worker {} distance to PrioQ {}: {}", origin, prioqcoord, distance_worker)

        temp_distance = distance_worker

        for worker in self._routepool.keys():
            if worker == origin or self._routepool[worker].has_prio_event \
                    or self._routepool[origin].last_round_prio_event:
                continue
            worker_pos = self._routepool[worker].current_pos
            prio_distance = get_distance_of_two_points_in_meters(worker_pos.lat, worker_pos.lng,
                                                                 prioqcoord.lat, prioqcoord.lng)
            logger.debug("Worker {} distance to PrioQ {}: {}", worker, prioqcoord, prio_distance)
            if prio_distance < temp_distance:
                logger.debug("Worker {} closer then {} by {} meters",
                             worker,
                             origin,
                             int(distance_worker) - int(prio_distance))
                temp_distance = prio_distance
                closer_worker = worker

        if closer_worker is not None:
            with self._manager_mutex:
                self._routepool[closer_worker].has_prio_event = True
                self._routepool[closer_worker].prio_coords = prioqcoord
            logger.debug("Worker {} is closer to PrioQ event {}", closer_worker, prioqcoord)
            return True

        logger.debug("No Worker is closer to PrioQ event {} than {}", prioqcoord, origin)

        return False

    # to be called regularly to remove inactive workers that used to be registered
    def _check_routepools(self, timeout: int = 300):
        while not self._stop_update_thread.is_set():
            routepool_changed: bool = False
            logger.debug("Checking routepool for idle/dead workers")
            with self._manager_mutex:
                for origin in list(self._routepool):
                    entry: RoutePoolEntry = self._routepool[origin]
                    if time.time() - entry.last_access > timeout + entry.worker_sleeping:
                        logger.warning(
                            "Worker {} has not accessed a location in {} seconds, removing from routemanager",
                            origin, timeout)
                        self.unregister_worker(origin)

            i = 0
            while i < 60 and not self._stop_update_thread.is_set():
                if self._stop_update_thread.is_set():
                    logger.info("Stop checking routepools")
                    break
                i += 1
                time.sleep(1)

    def set_worker_sleeping(self, origin: str, sleep_duration: float):
        if sleep_duration > 0:
            with self._manager_mutex:
                if origin in self._routepool:
                    self._routepool[origin].worker_sleeping = sleep_duration

    def __worker_changed_update_routepools(self):
        less_coords: bool = False
        if not self._is_started:
            return True
        temp_coordplist: List[Location] = []
        if self.mode in ("iv_mitm", "idle"):
            logger.info('Not updating routepools in iv_mitm mode')
            return True
        with self._manager_mutex:
            logger.debug("Updating all routepools")
            if len(self._workers_registered) == 0:
                logger.info("No registered workers, aborting __worker_changed_update_routepools...")
                return False

            if len(self._workers_registered) > len(self._current_route_round_coords):
                less_coords = True
                new_subroute_length = len(self._current_route_round_coords)
            else:
                try:
                    new_subroute_length = math.floor(len(self._current_route_round_coords) /
                                                     len(self._workers_registered))
                    if new_subroute_length == 0:
                        return False
                except Exception as e:
                    logger.info('Something happens with the worker - breakup')
                    return False
            i: int = 0
            temp_total_round: collections.deque = collections.deque(self._current_route_round_coords)

            # we want to order the dict by the time's we added the workers to the areas
            # we first need to build a list of tuples with only origin, time_added
            logger.debug("Checking routepool: {}", self._routepool)
            reduced_routepools = [(origin, self._routepool[origin].time_added) for origin in self._routepool]
            sorted_routepools = sorted(reduced_routepools, key=itemgetter(1))

            logger.debug("Checking routepools in the following order: {}", sorted_routepools)
            compare = lambda x, y: collections.Counter(x) == collections.Counter(y)
            for origin, time_added in sorted_routepools:
                entry: RoutePoolEntry = self._routepool[origin]
                logger.debug("Checking subroute of {}", origin)
                # let's assume a worker has already been removed or added to the dict (keys)...

                new_subroute: List[Location] = []
                j: int = 0
                while len(temp_total_round) > 0 and (j <= new_subroute_length or i == len(self._routepool)):
                    j += 1
                    new_subroute.append(temp_total_round.popleft())

                i += 1
                if len(entry.subroute) == 0:
                    logger.debug("{}'s subroute is empty, assuming he has freshly registered and desperately needs a "
                                 "queue", origin)
                    # worker is freshly registering, pass him his fair share
                    entry.subroute = new_subroute
                    # let's clean the queue just to make sure
                    entry.queue.clear()
                elif len(new_subroute) == len(entry.subroute):
                    logger.debug("{}'s subroute is as long as the old one, we will assume it hasn't changed (for now)",
                                 origin)
                    # apparently nothing changed
                    if compare(new_subroute, entry.subroute):
                        logger.info("Apparently no changes in subroutes...")
                    else:
                        logger.info("Subroute of {} has changed. Replacing entirely", origin)
                        # TODO: what now?
                        logger.debug('new_subroute: {}', new_subroute)
                        logger.debug('entry.subroute: {}', entry.subroute)
                        logger.debug('new_subroute == entry.subroute: {}', new_subroute == entry.subroute)
                        entry.subroute = new_subroute
                elif len(new_subroute) == 0:
                    logger.info("New subroute of {} is empty...", origin)
                    entry.subroute = new_subroute
                elif len(entry.subroute) > len(new_subroute) > 0:
                    logger.debug("{}'s subroute is longer than it should be now (maybe a worker has been "
                                 "added)", origin)
                    # we apparently have added at least a worker...
                    #   1) reduce the start of the current queue to start of new route
                    #   2) append the coords missing (check end of old routelength, add/remove from there on compared
                    #      to new)
                    old_queue: collections.deque = collections.deque(entry.queue)
                    while len(old_queue) > 0 and len(new_subroute) > 0 and old_queue.popleft() != new_subroute[0]:
                        pass

                    if len(old_queue) == 0:
                        logger.debug("{}'s queue is empty, we can just pass him the new subroute", origin)
                        # just set new route...
                        entry.queue = collections.deque()
                        for location in new_subroute:
                            entry.queue.append(location)
                    else:
                        # we now are at a point where we need to also check the end of the old queue and
                        # append possibly missing coords to it
                        logger.debug("Checking if the last element of the old queue is present in new subroute")
                        last_el_old_q: Location = old_queue[len(old_queue) - 1]
                        if last_el_old_q in new_subroute:
                            # we have the last element in the old subroute, we can actually append stuff with the
                            # diff to the new route
                            logger.debug("Last element of old queue is present in new subroute, appending the rest of "
                                         "the new subroute to the queue")
                            new_subroute_copy = collections.deque(new_subroute)
                            while len(new_subroute_copy) > 0 and new_subroute_copy.popleft() != last_el_old_q:
                                pass
                            logger.debug("Length of subroute to be extended by {}", len(new_subroute_copy))
                            # replace queue with old_queue
                            entry.queue.clear()
                            entry.queue = old_queue
                            while len(new_subroute_copy) > 0:
                                entry.queue.append(new_subroute_copy.popleft())
                        else:
                            # clear old route and replace with new_subroute
                            # maybe the worker jumps a wider distance
                            entry.queue.clear()
                            new_subroute_copy = collections.deque(new_subroute)
                            while len(new_subroute_copy) > 0:
                                entry.queue.append(new_subroute_copy.popleft())

                elif len(new_subroute) > len(entry.subroute) > 0:
                    #   old routelength < new len(route)/n:
                    #   we have likely removed a worker and need to redistribute
                    #   1) fetch start and end of old queue
                    #   2) we sorta ignore start/what's been visited so far
                    #   3) if the end is not part of the new route, check for the last coord of the current route
                    #   still in
                    #   the new route, remove the old rest of it (or just fetch the first coord of the next subroute and
                    #   remove the coords of that coord onward)
                    logger.debug("A worker has apparently been removed from the routepool")
                    last_el_old_route: Location = entry.subroute[len(entry.subroute) - 1]
                    old_queue_list: List[Location] = list(entry.queue)
                    old_queue: collections.deque = collections.deque(entry.queue)

                    last_el_new_route: Location = new_subroute[len(new_subroute) - 1]
                    # check last element of new subroute:
                    if last_el_new_route is not None and last_el_new_route in old_queue_list:
                        # if in current queue, remove from end of new subroute to end of old queue
                        logger.debug("Last element of new subroute is in old queue, removing everything after that "
                                     "element")
                        del old_queue_list[old_queue.index(last_el_new_route): len(old_queue_list) - 1]
                    elif last_el_old_route in new_subroute:
                        # append from end of queue (compared to new subroute) to end of new subroute
                        logger.debug("Last element of old queue in new subroute, appending everything afterwards")
                        missing_new_route_part: List[Location] = new_subroute.copy()
                        del missing_new_route_part[0: new_subroute.index(last_el_old_route)]
                        old_queue_list.extend(missing_new_route_part)

                    entry.queue = collections.deque()
                    [entry.queue.append(i) for i in old_queue_list]

                if len(entry.queue) == 0:
                    [entry.queue.append(i) for i in new_subroute]
                # don't forget to update the subroute ;)
                entry.subroute = new_subroute

                if less_coords:
                    new_subroute_length = 0

            logger.debug("Done updating subroutes")
            return True
            # TODO: A worker has been removed or added, we need to update the individual workerpools/queues
            #
            # First: Split the original route by the remaining workers => we have a list of new subroutes of
            # len(route)/n coordinates
            #
            # Iterate over all remaining routepools
            # Possible situations now:
            #
            #   Routelengths == new len(route)/n:
            #   Apparently nothing has changed...
            #
            #   old routelength > new len(route)/n:
            #   we have likely added a worker and need to redistribute
            #   1) reduce the start of the current queue to start after the end of the previous pool
            #   2) append the coords missing (check end of old routelength, add/remove from there on compared to new)

            #
            #   old routelength < new len(route)/n:
            #   we have likely removed a worker and need to redistribute
            #   1) fetch start and end of old queue
            #   2) we sorta ignore start/what's been visited so far
            #   3) if the end is not part of the new route, check for the last coord of the current route still in
            #   the new route, remove the old rest of it (or just fetch the first coord of the next subroute and
            #   remove the coords of that coord onward)

    def change_init_mapping(self, name_area: str):
        update = {
            'init': False
        }
        self._data_manager.set_data('area', 'patch', update, identifier=self.area_id)

    def get_route_status(self, origin) -> Tuple[int, int]:
        if self._route and origin in self._routepool:
            entry: RoutePoolEntry = self._routepool[origin]
            return len(entry.subroute) - len(entry.queue), len(entry.subroute)
        return 1, 1

    def get_rounds(self, origin: str) -> int:
        return self.check_worker_rounds()

    def get_registered_workers(self) -> int:
        self._workers_registered_mutex.acquire()
        try:
            return len(self._workers_registered)
        finally:
            self._workers_registered_mutex.release()

    def get_position_type(self, origin: str) -> Optional[str]:
        return self._positiontyp.get(origin, None)

    def get_geofence_helper(self) -> Optional[GeofenceHelper]:
        return self.geofence_helper

    def get_init(self) -> bool:
        return self.init

    def get_mode(self):
        return self.mode

    def get_settings(self) -> Optional[dict]:
        return self.settings

    def get_current_route(self) -> List[Location]:
        return self._route

    def get_current_prioroute(self) -> List[Location]:
        return self._prio_queue

    def get_level_mode(self):
        return self._level

    def redo_stop(self, worker, lat, lon):
        logger.info('Worker {} redo a unprocessed Stop ({}, {})'.format(str(worker), str(lat), str(lon)))
        if worker in self._routepool:
            self._routepool[worker].has_prio_event = True
            self._routepool[worker].prio_coords = Location(lat, lon)
            return True
        return False

    def get_coords_from_workers(self):
        coordlist: List[Location] = []
        logger.info('Getting all coords from workers')
        for origin in self._routepool:
            [coordlist.append(i) for i in self._routepool[origin].queue]

        logger.debug('Open Coords from workers: {}'.format(str(coordlist)))
        return coordlist
Exemple #7
0
class RouteManagerBase(ABC):
    def __init__(self, db_wrapper, coords, max_radius, max_coords_within_radius, path_to_include_geofence,
                 path_to_exclude_geofence, routefile, init=False,
                 name="unknown", settings=None):
        self.db_wrapper = db_wrapper
        self.init = init
        self.name = name
        self._coords_unstructured = coords
        self.geofence_helper = GeofenceHelper(path_to_include_geofence, path_to_exclude_geofence)
        self._routefile = routefile
        self._max_radius = max_radius
        self._max_coords_within_radius = max_coords_within_radius
        self.settings = settings

        self._last_round_prio = False
        self._manager_mutex = Lock()
        self._round_started_time = None
        if coords is not None:
            if init:
                fenced_coords = coords
            else:
                fenced_coords = self.geofence_helper.get_geofenced_coordinates(coords)
            self._route = getJsonRoute(fenced_coords, max_radius, max_coords_within_radius, routefile)
        else:
            self._route = None
        self._current_index_of_route = 0
        if settings is not None:
            self.delay_after_timestamp_prio = settings.get("delay_after_prio_event", None)
            self.starve_route = settings.get("starve_route", False)
        else:
            self.delay_after_timestamp_prio = None
            self.starve_route = False
        if self.delay_after_timestamp_prio is not None:
            self._prio_queue = []
            self.clustering_helper = ClusteringHelper(self._max_radius,
                                                      self._max_coords_within_radius,
                                                      self._cluster_priority_queue_criteria())
            self._stop_update_thread = Event()
            self._update_prio_queue_thread = Thread(name="prio_queue_update_" + name,
                                                    target=self._update_priority_queue_loop)
            self._update_prio_queue_thread.daemon = True
            self._update_prio_queue_thread.start()
        else:
            self._prio_queue = None

    def __del__(self):
        if self.delay_after_timestamp_prio:
            self._stop_update_thread.set()
            self._update_prio_queue_thread.join()

    def clear_coords(self):
        self._manager_mutex.acquire()
        self._coords_unstructured = None
        self._manager_mutex.release()

    # list_coords is a numpy array of arrays!
    def add_coords_numpy(self, list_coords):
        fenced_coords = self.geofence_helper.get_geofenced_coordinates(list_coords)
        self._manager_mutex.acquire()
        if self._coords_unstructured is None:
            self._coords_unstructured = fenced_coords
        else:
            self._coords_unstructured = np.concatenate((self._coords_unstructured, fenced_coords))
        self._manager_mutex.release()

    def add_coords_list(self, list_coords):
        to_be_appended = np.zeros(shape=(len(list_coords), 2))
        for i in range(len(list_coords)):
            to_be_appended[i][0] = list_coords[i][0]
            to_be_appended[i][1] = list_coords[i][1]
        self.add_coords_numpy(to_be_appended)

    @staticmethod
    def calculate_new_route(coords, max_radius, max_coords_within_radius, routefile, delete_old_route, num_procs=0):
        if delete_old_route and os.path.exists(routefile + ".calc"):
            log.debug("Deleting routefile...")
            os.remove(routefile + ".calc")
        new_route = getJsonRoute(coords, max_radius, max_coords_within_radius, num_processes=num_procs,
                                 routefile=routefile)
        return new_route

    def recalc_route(self, max_radius, max_coords_within_radius, num_procs=1, delete_old_route=False):
        current_coords = self._coords_unstructured
        routefile = self._routefile
        new_route = RouteManagerBase.calculate_new_route(current_coords, max_radius, max_coords_within_radius,
                                                         routefile, delete_old_route, num_procs)
        self._manager_mutex.acquire()
        self._route = new_route
        self._current_index_of_route = 0
        self._manager_mutex.release()

    def _update_priority_queue_loop(self):
        while not self._stop_update_thread.is_set():
            # retrieve the latest hatches from DB
            # newQueue = self._db_wrapper.get_next_raid_hatches(self._delayAfterHatch, self._geofenceHelper)
            new_queue = self._retrieve_latest_priority_queue()
            self._merge_priority_queue(new_queue)
            time.sleep(300)

    def _merge_priority_queue(self, new_queue):
        self._manager_mutex.acquire()
        merged = list(set(new_queue + self._prio_queue))
        merged = self._filter_priority_queue_internal(merged)
        heapq.heapify(merged)
        self._prio_queue = merged
        self._manager_mutex.release()
        log.info("New priorityqueue: %s" % merged)

    def date_diff_in_seconds(self, dt2, dt1):
        timedelta = dt2 - dt1
        return timedelta.days * 24 * 3600 + timedelta.seconds

    def dhms_from_seconds(self, seconds):
        minutes, seconds = divmod(seconds, 60)
        hours, minutes = divmod(minutes, 60)
        # days, hours = divmod(hours, 24)
        return hours, minutes, seconds

    def _get_round_finished_string(self):
        round_finish_time = datetime.now()
        round_completed_in = (
                "%d hours, %d minutes, %d seconds" % (
                    self.dhms_from_seconds(
                            self.date_diff_in_seconds(round_finish_time, self._round_started_time)
                        )
                    )
        )
        return round_completed_in

    @abstractmethod
    def _retrieve_latest_priority_queue(self):
        """
        Method that's supposed to return a plain list containing (timestamp, Location) of the next events of interest
        :return:
        """
        pass

    @abstractmethod
    def _get_coords_post_init(self):
        """
        Return list of coords to be fetched and used for routecalc
        :return:
        """
        pass

    @abstractmethod
    def _cluster_priority_queue_criteria(self):
        """
        If you do not want to have any filtering, simply return 0, 0, otherwise simply
        return timedelta_seconds, distance
        :return:
        """

    def _filter_priority_queue_internal(self, latest):
        """
        Filter through the internal priority queue and cluster events within the timedelta and distance returned by
        _cluster_priority_queue_criteria
        :return:
        """
        # timedelta_seconds = self._cluster_priority_queue_criteria()
        delete_seconds_passed = 0
        if self.settings is not None:
            delete_seconds_passed = self.settings.get("remove_from_queue_backlog", 0)

        if delete_seconds_passed is not None:
            delete_before = time.time() - delete_seconds_passed
        else:
            delete_before = 0
        latest = [to_keep for to_keep in latest if not to_keep[0] < delete_before]
        # TODO: sort latest by modified flag of event
        # merged = self._merge_queue(latest, self._max_radius, 2, timedelta_seconds)
        merged = self.clustering_helper.get_clustered(latest)
        return merged

    def get_next_location(self):
        log.debug("get_next_location of %s called" % str(self.name))
        next_lat, next_lng = 0, 0

        # first check if a location is available, if not, block until we have one...
        got_location = False
        while not got_location:
            log.debug("%s: Checking if a location is available..." % str(self.name))
            self._manager_mutex.acquire()
            got_location = self._prio_queue is not None and len(self._prio_queue) > 0 or len(self._route) > 0
            self._manager_mutex.release()
            if not got_location:
                log.debug("%s: No location available yet" % str(self.name))
                time.sleep(0.5)
        log.debug("%s: Location available, acquiring lock and trying to return location")
        self._manager_mutex.acquire()
        # check priority queue for items of priority that are past our time...
        # if that is not the case, simply increase the index in route and return the location on route

        # determine whether we move to the next location or the prio queue top's item
        if (self.delay_after_timestamp_prio is not None and ((not self._last_round_prio or self.starve_route)
                                                             and len(self._prio_queue) > 0
                                                             and self._prio_queue[0][0] < time.time())):
            log.debug("%s: Priority event" % str(self.name))
            next_stop = heapq.heappop(self._prio_queue)[1]
            next_lat = next_stop.lat
            next_lng = next_stop.lng
            self._last_round_prio = True
            log.info("Round of route %s is moving to %s, %s for a priority event"
                     % (str(self.name), str(next_lat), str(next_lng)))
        else:
            log.debug("%s: Moving on with route" % str(self.name))
            if self._current_index_of_route == 0:
                if self._round_started_time is not None:
                    log.info("Round of route %s reached the first spot again. It took %s"
                             % (str(self.name), str(self._get_round_finished_string())))
                self._round_started_time = datetime.now()
                log.info("Round of route %s started at %s" % (str(self.name), str(self._round_started_time)))

            # continue as usual
            log.info("Moving on with location %s" % self._route[self._current_index_of_route])
            next_lat = self._route[self._current_index_of_route]['lat']
            next_lng = self._route[self._current_index_of_route]['lng']
            self._current_index_of_route += 1
            if self.init and self._current_index_of_route >= len(self._route):
                # we are done with init, let's calculate a new route
                log.warning("Init of %s done, it took %s, calculating new route..."
                            % (str(self.name), self._get_round_finished_string()))
                self._manager_mutex.release()
                self.clear_coords()
                coords = self._get_coords_post_init()
                log.debug("Setting %s coords to as new points in route of %s"
                          % (str(len(coords)), str(self.name)))
                self.add_coords_list(coords)
                log.debug("Route of %s is being calculated" % str(self.name))
                self.recalc_route(self._max_radius, self._max_coords_within_radius, 1, True)
                self.init = False
                return self.get_next_location()
            elif self._current_index_of_route >= len(self._route):
                self._current_index_of_route = 0
            self._last_round_prio = False
        log.info("%s done grabbing next coord, releasing lock and returning location: %s, %s"
                 % (str(self.name), str(next_lat), str(next_lng)))
        self._manager_mutex.release()
        return Location(next_lat, next_lng)
Exemple #8
0
class RouteManagerBase(ABC):
    def __init__(self,
                 db_wrapper: DbWrapperBase,
                 coords: List[Location],
                 max_radius: float,
                 max_coords_within_radius: int,
                 path_to_include_geofence: str,
                 path_to_exclude_geofence: str,
                 routefile: str,
                 mode=None,
                 init: bool = False,
                 name: str = "unknown",
                 settings: dict = None):
        self.db_wrapper: DbWrapperBase = db_wrapper
        self.init: bool = init
        self.name: str = name
        self._coords_unstructured: List[Location] = coords
        self.geofence_helper: GeofenceHelper = GeofenceHelper(
            path_to_include_geofence, path_to_exclude_geofence)
        self._routefile = os.path.join(args.file_path, routefile)
        self._max_radius: float = max_radius
        self._max_coords_within_radius: int = max_coords_within_radius
        self.settings: dict = settings
        self.mode = mode
        self._is_started: bool = False
        self._first_started = False
        self._route_queue = Queue()
        self._start_calc: bool = False
        self._rounds = {}
        self._positiontyp = {}
        self._coords_to_be_ignored = set()

        # we want to store the workers using the routemanager
        self._workers_registered: List[WorkerBase] = []
        self._workers_registered_mutex = Lock()

        self._last_round_prio = {}
        self._manager_mutex = RLock()
        self._round_started_time = None
        self._route: List[Location] = []

        if coords is not None:
            if init:
                fenced_coords = coords
            else:
                fenced_coords = self.geofence_helper.get_geofenced_coordinates(
                    coords)
            new_coords = getJsonRoute(fenced_coords, max_radius,
                                      max_coords_within_radius, routefile)
            for coord in new_coords:
                self._route.append(Location(coord["lat"], coord["lng"]))
        self._current_index_of_route = 0
        self._init_mode_rounds = 0

        if self.settings is not None:
            self.delay_after_timestamp_prio = self.settings.get(
                "delay_after_prio_event", None)
            self.starve_route = self.settings.get("starve_route", False)
        else:
            self.delay_after_timestamp_prio = None
            self.starve_route = False

        # initialize priority queue variables
        self._prio_queue = None
        self._update_prio_queue_thread = None
        self._stop_update_thread = Event()

    def stop_routemanager(self):
        if self._update_prio_queue_thread is not None:
            self._stop_update_thread.set()
            self._update_prio_queue_thread.join()

    def _init_route_queue(self):
        self._manager_mutex.acquire()
        try:
            if len(self._route) > 0:
                self._route_queue.queue.clear()
                logger.debug("Creating queue for coords")
                for latlng in self._route:
                    self._route_queue.put((latlng.lat, latlng.lng))
                logger.debug("Finished creating queue")
        finally:
            self._manager_mutex.release()

    def clear_coords(self):
        self._manager_mutex.acquire()
        self._coords_unstructured = None
        self._manager_mutex.release()

    def register_worker(self, worker_name):
        self._workers_registered_mutex.acquire()
        try:
            if worker_name in self._workers_registered:
                logger.info("Worker {} already registered to routemanager {}",
                            str(worker_name), str(self.name))
                return False
            else:
                logger.info("Worker {} registering to routemanager {}",
                            str(worker_name), str(self.name))
                self._workers_registered.append(worker_name)
                self._rounds[worker_name] = 0
                self._positiontyp[worker_name] = 0

                return True
        finally:
            self._workers_registered_mutex.release()

    def unregister_worker(self, worker_name):
        self._workers_registered_mutex.acquire()
        try:
            if worker_name in self._workers_registered:
                logger.info("Worker {} unregistering from routemanager {}",
                            str(worker_name), str(self.name))
                self._workers_registered.remove(worker_name)
                del self._rounds[worker_name]
            else:
                # TODO: handle differently?
                logger.info(
                    "Worker {} failed unregistering from routemanager {} since subscription was previously lifted",
                    str(worker_name), str(self.name))
            if len(self._workers_registered) == 0 and self._is_started:
                logger.info(
                    "Routemanager {} does not have any subscribing workers anymore, calling stop",
                    str(self.name))
                self._quit_route()
        finally:
            self._workers_registered_mutex.release()

    def _check_started(self):
        return self._is_started

    def _start_priority_queue(self):
        if (self._update_prio_queue_thread is None
                and (self.delay_after_timestamp_prio is not None
                     or self.mode == "iv_mitm")
                and not self.mode == "pokestops"):
            self._prio_queue = []
            if self.mode not in ["iv_mitm", "pokestops"]:
                self.clustering_helper = ClusteringHelper(
                    self._max_radius, self._max_coords_within_radius,
                    self._cluster_priority_queue_criteria())
            self._update_prio_queue_thread = Thread(
                name="prio_queue_update_" + self.name,
                target=self._update_priority_queue_loop)
            self._update_prio_queue_thread.daemon = False
            self._update_prio_queue_thread.start()

    # list_coords is a numpy array of arrays!
    def add_coords_numpy(self, list_coords: np.ndarray):
        fenced_coords = self.geofence_helper.get_geofenced_coordinates(
            list_coords)
        self._manager_mutex.acquire()
        if self._coords_unstructured is None:
            self._coords_unstructured = fenced_coords
        else:
            self._coords_unstructured = np.concatenate(
                (self._coords_unstructured, fenced_coords))
        self._manager_mutex.release()

    def add_coords_list(self, list_coords: List[Location]):
        to_be_appended = np.zeros(shape=(len(list_coords), 2))
        for i in range(len(list_coords)):
            to_be_appended[i][0] = float(list_coords[i].lat)
            to_be_appended[i][1] = float(list_coords[i].lng)
        self.add_coords_numpy(to_be_appended)

    @staticmethod
    def calculate_new_route(coords,
                            max_radius,
                            max_coords_within_radius,
                            routefile,
                            delete_old_route,
                            num_procs=0):
        if delete_old_route and os.path.exists(str(routefile) + ".calc"):
            logger.debug("Deleting routefile...")
            os.remove(str(routefile) + ".calc")
        new_route = getJsonRoute(coords,
                                 max_radius,
                                 max_coords_within_radius,
                                 num_processes=num_procs,
                                 routefile=routefile)
        return new_route

    def empty_routequeue(self):
        return self._route_queue.empty()

    def recalc_route(self,
                     max_radius: float,
                     max_coords_within_radius: int,
                     num_procs: int = 1,
                     delete_old_route: bool = False,
                     nofile: bool = False):
        current_coords = self._coords_unstructured
        if nofile:
            routefile = None
        else:
            routefile = self._routefile
        new_route = RouteManagerBase.calculate_new_route(
            current_coords, max_radius, max_coords_within_radius, routefile,
            delete_old_route, num_procs)
        self._manager_mutex.acquire()
        self._route.clear()
        for coord in new_route:
            self._route.append(Location(coord["lat"], coord["lng"]))
        self._current_index_of_route = 0
        self._manager_mutex.release()

    def _update_priority_queue_loop(self):
        if self._priority_queue_update_interval(
        ) is None or self._priority_queue_update_interval() == 0:
            return
        while not self._stop_update_thread.is_set():
            # retrieve the latest hatches from DB
            # newQueue = self._db_wrapper.get_next_raid_hatches(self._delayAfterHatch, self._geofenceHelper)
            new_queue = self._retrieve_latest_priority_queue()
            self._merge_priority_queue(new_queue)
            time.sleep(self._priority_queue_update_interval())

    def _merge_priority_queue(self, new_queue):
        if new_queue is not None:
            self._manager_mutex.acquire()
            merged = list(new_queue)
            logger.info("New raw priority queue with {} entries", len(merged))
            merged = self._filter_priority_queue_internal(merged)
            heapq.heapify(merged)
            self._prio_queue = merged
            self._manager_mutex.release()
            logger.info("New clustered priority queue with {} entries",
                        len(merged))
            logger.debug("Priority queue entries: {}", str(merged))

    def date_diff_in_seconds(self, dt2, dt1):
        timedelta = dt2 - dt1
        return timedelta.days * 24 * 3600 + timedelta.seconds

    def dhms_from_seconds(self, seconds):
        minutes, seconds = divmod(seconds, 60)
        hours, minutes = divmod(minutes, 60)
        # days, hours = divmod(hours, 24)
        return hours, minutes, seconds

    def _get_round_finished_string(self):
        round_finish_time = datetime.now()
        round_completed_in = (
            "%d hours, %d minutes, %d seconds" % (self.dhms_from_seconds(
                self.date_diff_in_seconds(round_finish_time,
                                          self._round_started_time))))
        return round_completed_in

    def add_coord_to_be_removed(self, lat: float, lon: float):
        if lat < -90.0 or lat > 90.0 or lon < -180.0 or lon > 180.0:
            return
        self._manager_mutex.acquire()
        self._coords_to_be_ignored.add(Location(lat, lon))
        self._manager_mutex.release()

    @abstractmethod
    def _retrieve_latest_priority_queue(self):
        """
        Method that's supposed to return a plain list containing (timestamp, Location) of the next events of interest
        :return:
        """
        pass

    @abstractmethod
    def _start_routemanager(self):
        """
        Starts priority queue or whatever the implementations require
        :return:
        """
        pass

    @abstractmethod
    def _quit_route(self):
        """
        Killing the Route Thread
        :return:
        """
        pass

    @abstractmethod
    def _get_coords_post_init(self):
        """
        Return list of coords to be fetched and used for routecalc
        :return:
        """
        pass

    @abstractmethod
    def _check_coords_before_returning(self, lat, lng):
        """
        Return list of coords to be fetched and used for routecalc
        :return:
        """
        pass

    @abstractmethod
    def _recalc_route_workertype(self):
        """
        Return a new route for worker
        :return:
        """
        pass

    @abstractmethod
    def _get_coords_after_finish_route(self):
        """
        Return list of coords to be fetched after finish a route
        :return:
        """
        pass

    @abstractmethod
    def _cluster_priority_queue_criteria(self):
        """
        If you do not want to have any filtering, simply return 0, 0, otherwise simply
        return timedelta_seconds, distance
        :return:
        """

    @abstractmethod
    def _priority_queue_update_interval(self):
        """
        The time to sleep in between consecutive updates of the priority queue
        :return:
        """

    def _filter_priority_queue_internal(self, latest):
        """
        Filter through the internal priority queue and cluster events within the timedelta and distance returned by
        _cluster_priority_queue_criteria
        :return:
        """
        # timedelta_seconds = self._cluster_priority_queue_criteria()
        if self.mode == "iv_mitm":
            # exclude IV prioQ to also pass encounterIDs since we do not pass additional information through when
            # clustering
            return latest
        delete_seconds_passed = 0
        if self.settings is not None:
            delete_seconds_passed = self.settings.get(
                "remove_from_queue_backlog", 0)

        if delete_seconds_passed is not None:
            delete_before = time.time() - delete_seconds_passed
        else:
            delete_before = 0
        latest = [
            to_keep for to_keep in latest if not to_keep[0] < delete_before
        ]
        # TODO: sort latest by modified flag of event
        # merged = self._merge_queue(latest, self._max_radius, 2, timedelta_seconds)
        merged = self.clustering_helper.get_clustered(latest)
        return merged

    def get_next_location(self, origin):
        logger.debug("get_next_location of {} called", str(self.name))
        if not self._is_started:
            logger.info("Starting routemanager {} in get_next_location",
                        str(self.name))
            self._start_routemanager()
        next_lat, next_lng = 0, 0

        if self._start_calc:
            logger.info("Another process already calculate the new route")
            return None

        # first check if a location is available, if not, block until we have one...
        got_location = False
        while not got_location and self._is_started and not self.init:
            logger.debug("{}: Checking if a location is available...",
                         str(self.name))
            self._manager_mutex.acquire()
            got_location = not self._route_queue.empty() or (
                self._prio_queue is not None and len(self._prio_queue) > 0)
            self._manager_mutex.release()
            if not got_location:
                logger.debug("{}: No location available yet", str(self.name))
                if self._get_coords_after_finish_route() and not self.init:
                    # getting new coords or IV worker
                    time.sleep(1)
                else:
                    logger.info("Not getting new coords - leaving worker")
                    return None

        logger.debug(
            "{}: Location available, acquiring lock and trying to return location",
            str(self.name))
        self._manager_mutex.acquire()
        # check priority queue for items of priority that are past our time...
        # if that is not the case, simply increase the index in route and return the location on route

        # determine whether we move to the next location or the prio queue top's item
        if (self.delay_after_timestamp_prio is not None and (
            (not self._last_round_prio.get(origin, False) or self.starve_route)
                and self._prio_queue and len(self._prio_queue) > 0
                and self._prio_queue[0][0] < time.time())):
            logger.debug("{}: Priority event", str(self.name))
            next_stop = heapq.heappop(self._prio_queue)[1]
            next_lat = next_stop.lat
            next_lng = next_stop.lng
            self._last_round_prio[origin] = True
            self._positiontyp[origin] = 1
            logger.info(
                "Round of route {} is moving to {}, {} for a priority event",
                str(self.name), str(next_lat), str(next_lng))
        else:
            logger.debug("{}: Moving on with route", str(self.name))
            self._positiontyp[origin] = 0
            if len(self._route) == self._route_queue.qsize():
                if self._round_started_time is not None:
                    logger.info(
                        "Round of route {} reached the first spot again. It took {}",
                        str(self.name), str(self._get_round_finished_string()))
                    self.add_route_to_origin()
                self._round_started_time = datetime.now()
                if len(self._route) == 0:
                    return None
                logger.info("Round of route {} started at {}", str(self.name),
                            str(self._round_started_time))
            elif self._round_started_time is None:
                self._round_started_time = datetime.now()

            # continue as usual

            if self.init and (self._route_queue.empty()):
                self._init_mode_rounds += 1
            if self.init and (self._route_queue.empty()) and \
                    self._init_mode_rounds >= int(self.settings.get("init_mode_rounds", 1)):
                # we are done with init, let's calculate a new route
                logger.warning(
                    "Init of {} done, it took {}, calculating new route...",
                    str(self.name), self._get_round_finished_string())
                if self._start_calc:
                    logger.info(
                        "Another process already calculate the new route")
                    self._manager_mutex.release()
                    return None
                self._start_calc = True
                self.clear_coords()
                coords = self._get_coords_post_init()
                logger.debug(
                    "Setting {} coords to as new points in route of {}",
                    str(len(coords)), str(self.name))
                self.add_coords_list(coords)
                logger.debug("Route of {} is being calculated", str(self.name))
                self._recalc_route_workertype()
                self.init = False
                self.change_init_mapping(self.name)
                self._manager_mutex.release()
                self._start_calc = False
                logger.debug("Initroute of {} is finished - restart worker",
                             str(self.name))
                return None
            elif (self._route_queue.qsize()) == 1:
                logger.info('Reaching last coord of route')
            elif self._route_queue.empty():
                # normal queue is empty - prioQ is filled. Try to generate a new Q
                logger.info("Normal routequeue is empty - try to fill up")
                if self._get_coords_after_finish_route():
                    # getting new coords or IV worker
                    self._manager_mutex.release()
                    return self.get_next_location(origin)
                elif not self._get_coords_after_finish_route():
                    logger.info("Not getting new coords - leaving worker")
                    self._manager_mutex.release()
                    return None
                self._manager_mutex.release()

            # getting new coord
            next_coord = self._route_queue.get()
            next_lat = next_coord[0]
            next_lng = next_coord[1]
            self._route_queue.task_done()
            logger.info("{}: Moving on with location {} [{} coords left]",
                        str(self.name), str(next_coord),
                        str(self._route_queue.qsize()))

            self._last_round_prio[origin] = False
        logger.debug(
            "{}: Done grabbing next coord, releasing lock and returning location: {}, {}",
            str(self.name), str(next_lat), str(next_lng))
        self._manager_mutex.release()
        if self._check_coords_before_returning(next_lat, next_lng):
            return Location(next_lat, next_lng)
        else:
            return self.get_next_location(origin)

    def del_from_route(self):
        logger.debug(
            "{}: Location available, acquiring lock and trying to return location",
            str(self.name))
        self._manager_mutex.acquire()
        logger.info('Removing coords from Route')
        self._route.pop(int(self._current_index_of_route) - 1)
        self._current_index_of_route -= 1
        if len(self._route) == 0:
            logger.info('No more coords are available... Sleeping.')
        self._manager_mutex.release()

    def change_init_mapping(self, name_area):
        with open('configs/mappings.json') as f:
            vars = json.load(f)

        for var in vars['areas']:
            if (var['name']) == name_area:
                var['init'] = bool(False)

        with open('configs/mappings.json', 'w') as outfile:
            json.dump(vars, outfile, indent=4, sort_keys=True)

    def get_route_status(self):
        if self._route:
            return (len(self._route) - self._route_queue.qsize()), len(
                self._route)
        return 1, 1

    def get_rounds(self, origin):
        return self._rounds.get(origin, 999)

    def add_route_to_origin(self):
        for origin in self._rounds:
            self._rounds[origin] += 1

    def get_registered_workers(self):
        return len(self._workers_registered)

    def get_position_type(self, origin):
        return self._positiontyp[origin]

    def get_walker_type(self):
        return self.mode
Exemple #9
0
class RouteManagerBase(ABC):
    def __init__(self,
                 db_wrapper,
                 coords,
                 max_radius,
                 max_coords_within_radius,
                 path_to_include_geofence,
                 path_to_exclude_geofence,
                 routefile,
                 mode=None,
                 init=False,
                 name="unknown",
                 settings=None):
        self.db_wrapper = db_wrapper
        self.init = init
        self.name = name
        self._coords_unstructured = coords
        self.geofence_helper = GeofenceHelper(path_to_include_geofence,
                                              path_to_exclude_geofence)
        self._routefile = routefile
        self._max_radius = max_radius
        self._max_coords_within_radius = max_coords_within_radius
        self.settings = settings
        self.mode = mode
        self._is_started = False

        # we want to store the workers using the routemanager
        self._workers_registered = []
        self._workers_registered_mutex = Lock()

        self._last_round_prio = False
        self._manager_mutex = RLock()
        self._round_started_time = None
        if coords is not None:
            if init:
                fenced_coords = coords
            else:
                fenced_coords = self.geofence_helper.get_geofenced_coordinates(
                    coords)
            self._route = getJsonRoute(fenced_coords, max_radius,
                                       max_coords_within_radius, routefile)
        else:
            self._route = None
        self._current_index_of_route = 0
        self._init_mode_rounds = 0

        if self.settings is not None:
            self.delay_after_timestamp_prio = self.settings.get(
                "delay_after_prio_event", None)
            self.starve_route = self.settings.get("starve_route", False)
        else:
            self.delay_after_timestamp_prio = None
            self.starve_route = False

        # initialize priority queue variables
        self._prio_queue = None
        self._update_prio_queue_thread = None
        self._stop_update_thread = Event()

    def __del__(self):
        if self._update_prio_queue_thread is not None:
            self._stop_update_thread.set()
            self._update_prio_queue_thread.join()

    def clear_coords(self):
        self._manager_mutex.acquire()
        self._coords_unstructured = None
        self._manager_mutex.release()

    def register_worker(self, worker_name):
        self._workers_registered_mutex.acquire()
        try:
            if worker_name in self._workers_registered:
                log.info("Worker %s already registered to routemanager %s" %
                         (str(worker_name), str(self.name)))
                return False
            else:
                log.info("Worker %s registering to routemanager %s" %
                         (str(worker_name), str(self.name)))
                self._workers_registered.append(worker_name)
                return True
        finally:
            self._workers_registered_mutex.release()

    def unregister_worker(self, worker_name):
        self._workers_registered_mutex.acquire()
        try:
            if worker_name in self._workers_registered:
                log.info("Worker %s unregistering from routemanager %s" %
                         (str(worker_name), str(self.name)))
                self._workers_registered.remove(worker_name)
            else:
                # TODO: handle differently?
                log.info(
                    "Worker %s failed unregistering from routemanager %s since subscription was previously "
                    "lifted" % (str(worker_name), str(self.name)))
            if len(self._workers_registered) == 0 and self._is_started:
                log.info(
                    "Routemanager %s does not have any subscribing workers anymore, calling stop"
                    % str(self.name))
                self._quit_route()
        finally:
            self._workers_registered_mutex.release()

    def _check_started(self):
        return self._is_started

    def _start_priority_queue(self):
        if (self._update_prio_queue_thread is None
                and (self.delay_after_timestamp_prio is not None
                     or self.mode == "iv_mitm")
                and not self.mode == "pokestops"):
            self._prio_queue = []
            if self.mode not in ["iv_mitm", "pokestops"]:
                self.clustering_helper = ClusteringHelper(
                    self._max_radius, self._max_coords_within_radius,
                    self._cluster_priority_queue_criteria())
            self._update_prio_queue_thread = Thread(
                name="prio_queue_update_" + self.name,
                target=self._update_priority_queue_loop)
            self._update_prio_queue_thread.daemon = False
            self._update_prio_queue_thread.start()

    # list_coords is a numpy array of arrays!
    def add_coords_numpy(self, list_coords):
        fenced_coords = self.geofence_helper.get_geofenced_coordinates(
            list_coords)
        self._manager_mutex.acquire()
        if self._coords_unstructured is None:
            self._coords_unstructured = fenced_coords
        else:
            self._coords_unstructured = np.concatenate(
                (self._coords_unstructured, fenced_coords))
        self._manager_mutex.release()

    def add_coords_list(self, list_coords):
        to_be_appended = np.zeros(shape=(len(list_coords), 2))
        for i in range(len(list_coords)):
            to_be_appended[i][0] = list_coords[i][0]
            to_be_appended[i][1] = list_coords[i][1]
        self.add_coords_numpy(to_be_appended)

    @staticmethod
    def calculate_new_route(coords,
                            max_radius,
                            max_coords_within_radius,
                            routefile,
                            delete_old_route,
                            num_procs=0):
        if delete_old_route and os.path.exists(routefile + ".calc"):
            log.debug("Deleting routefile...")
            os.remove(routefile + ".calc")
        new_route = getJsonRoute(coords,
                                 max_radius,
                                 max_coords_within_radius,
                                 num_processes=num_procs,
                                 routefile=routefile)
        return new_route

    def recalc_route(self,
                     max_radius,
                     max_coords_within_radius,
                     num_procs=1,
                     delete_old_route=False,
                     nofile=False):
        current_coords = self._coords_unstructured
        if nofile:
            routefile = None
        else:
            routefile = self._routefile
        new_route = RouteManagerBase.calculate_new_route(
            current_coords, max_radius, max_coords_within_radius, routefile,
            delete_old_route, num_procs)
        self._manager_mutex.acquire()
        self._route = new_route
        self._current_index_of_route = 0
        self._manager_mutex.release()

    def _update_priority_queue_loop(self):
        if self._priority_queue_update_interval(
        ) is None or self._priority_queue_update_interval() == 0:
            return
        while not self._stop_update_thread.is_set():
            # retrieve the latest hatches from DB
            # newQueue = self._db_wrapper.get_next_raid_hatches(self._delayAfterHatch, self._geofenceHelper)
            new_queue = self._retrieve_latest_priority_queue()
            self._merge_priority_queue(new_queue)
            time.sleep(self._priority_queue_update_interval())

    def _merge_priority_queue(self, new_queue):
        if new_queue is not None:
            self._manager_mutex.acquire()
            merged = set(new_queue + self._prio_queue)
            merged = list(merged)
            merged = self._filter_priority_queue_internal(merged)
            heapq.heapify(merged)
            self._prio_queue = merged
            self._manager_mutex.release()
            log.info("New priorityqueue: %s" % merged)

    def date_diff_in_seconds(self, dt2, dt1):
        timedelta = dt2 - dt1
        return timedelta.days * 24 * 3600 + timedelta.seconds

    def dhms_from_seconds(self, seconds):
        minutes, seconds = divmod(seconds, 60)
        hours, minutes = divmod(minutes, 60)
        # days, hours = divmod(hours, 24)
        return hours, minutes, seconds

    def _get_round_finished_string(self):
        round_finish_time = datetime.now()
        round_completed_in = (
            "%d hours, %d minutes, %d seconds" % (self.dhms_from_seconds(
                self.date_diff_in_seconds(round_finish_time,
                                          self._round_started_time))))
        return round_completed_in

    @abstractmethod
    def _retrieve_latest_priority_queue(self):
        """
        Method that's supposed to return a plain list containing (timestamp, Location) of the next events of interest
        :return:
        """
        pass

    @abstractmethod
    def _start_routemanager(self):
        """
        Starts priority queue or whatever the implementations require
        :return:
        """
        pass

    @abstractmethod
    def _quit_route(self):
        """
        Killing the Route Thread
        :return:
        """
        pass

    @abstractmethod
    def _get_coords_post_init(self):
        """
        Return list of coords to be fetched and used for routecalc
        :return:
        """
        pass

    @abstractmethod
    def _check_coords_before_returning(self, lat, lng):
        """
        Return list of coords to be fetched and used for routecalc
        :return:
        """
        pass

    @abstractmethod
    def _recalc_route_workertype(self):
        """
        Return a new route for worker
        :return:
        """
        pass

    @abstractmethod
    def _get_coords_after_finish_route(self):
        """
        Return list of coords to be fetched after finish a route
        :return:
        """
        pass

    @abstractmethod
    def _cluster_priority_queue_criteria(self):
        """
        If you do not want to have any filtering, simply return 0, 0, otherwise simply
        return timedelta_seconds, distance
        :return:
        """

    @abstractmethod
    def _priority_queue_update_interval(self):
        """
        The time to sleep in between consecutive updates of the priority queue
        :return:
        """

    def _filter_priority_queue_internal(self, latest):
        """
        Filter through the internal priority queue and cluster events within the timedelta and distance returned by
        _cluster_priority_queue_criteria
        :return:
        """
        # timedelta_seconds = self._cluster_priority_queue_criteria()
        if self.mode == "iv_mitm":
            # exclude IV prioQ to also pass encounterIDs since we do not pass additional information through when
            # clustering
            return latest
        delete_seconds_passed = 0
        if self.settings is not None:
            delete_seconds_passed = self.settings.get(
                "remove_from_queue_backlog", 0)

        if delete_seconds_passed is not None:
            delete_before = time.time() - delete_seconds_passed
        else:
            delete_before = 0
        latest = [
            to_keep for to_keep in latest if not to_keep[0] < delete_before
        ]
        # TODO: sort latest by modified flag of event
        # merged = self._merge_queue(latest, self._max_radius, 2, timedelta_seconds)
        merged = self.clustering_helper.get_clustered(latest)
        return merged

    def get_next_location(self):
        log.debug("get_next_location of %s called" % str(self.name))
        if not self._is_started:
            log.info("Starting routemanager %s in get_next_location" %
                     str(self.name))
            self._start_routemanager()
        next_lat, next_lng = 0, 0

        # first check if a location is available, if not, block until we have one...
        got_location = False
        while not got_location:
            log.debug("%s: Checking if a location is available..." %
                      str(self.name))
            self._manager_mutex.acquire()
            got_location = (self._prio_queue is not None
                            and len(self._prio_queue) > 0 or
                            (self._route is not None and len(self._route) > 0))
            self._manager_mutex.release()
            if not got_location:
                log.debug("%s: No location available yet" % str(self.name))
                time.sleep(0.5)
        log.debug(
            "%s: Location available, acquiring lock and trying to return location"
            % str(self.name))
        self._manager_mutex.acquire()
        # check priority queue for items of priority that are past our time...
        # if that is not the case, simply increase the index in route and return the location on route

        # determine whether we move to the next location or the prio queue top's item
        if (self.delay_after_timestamp_prio is not None
                and ((not self._last_round_prio or self.starve_route)
                     and self._prio_queue and len(self._prio_queue) > 0
                     and self._prio_queue[0][0] < time.time())):
            log.debug("%s: Priority event" % str(self.name))
            next_stop = heapq.heappop(self._prio_queue)[1]
            next_lat = next_stop.lat
            next_lng = next_stop.lng
            self._last_round_prio = True
            log.info(
                "Round of route %s is moving to %s, %s for a priority event" %
                (str(self.name), str(next_lat), str(next_lng)))
        else:
            log.debug("%s: Moving on with route" % str(self.name))
            if self._current_index_of_route == 0:
                if self._round_started_time is not None:
                    log.info(
                        "Round of route %s reached the first spot again. It took %s"
                        % (str(self.name),
                           str(self._get_round_finished_string())))
                self._round_started_time = datetime.now()
                if len(self._route) == 0: return None
                log.info("Round of route %s started at %s" %
                         (str(self.name), str(self._round_started_time)))

            # continue as usual
            if self._current_index_of_route < len(self._route):
                log.info(
                    "%s: Moving on with location %s [%s/%s]" %
                    (str(self.name), self._route[self._current_index_of_route],
                     self._current_index_of_route + 1, len(self._route)))
                next_lat = self._route[self._current_index_of_route]['lat']
                next_lng = self._route[self._current_index_of_route]['lng']
            self._current_index_of_route += 1
            if self.init and self._current_index_of_route >= len(self._route):
                self._init_mode_rounds += 1
            if self.init and self._current_index_of_route >= len(self._route) and \
                    self._init_mode_rounds >= int(self.settings.get("init_mode_rounds", 1)):
                # we are done with init, let's calculate a new route
                log.warning(
                    "Init of %s done, it took %s, calculating new route..." %
                    (str(self.name), self._get_round_finished_string()))
                self.clear_coords()
                coords = self._get_coords_post_init()
                log.debug("Setting %s coords to as new points in route of %s" %
                          (str(len(coords)), str(self.name)))
                self.add_coords_list(coords)
                log.debug("Route of %s is being calculated" % str(self.name))
                self._recalc_route_workertype()
                self.init = False
                self.change_init_mapping(self.name)
                self._manager_mutex.release()
                return self.get_next_location()
            elif self._current_index_of_route == len(self._route):
                log.info('Reaching last coord of route')
            elif self._current_index_of_route > len(self._route):
                self._current_index_of_route = 0
                coords_after_round = self._get_coords_after_finish_route()
                # TODO: check isinstance list?
                if coords_after_round is not None:
                    self.clear_coords()
                    coords = coords_after_round
                    self.add_coords_list(coords)
                    self._recalc_route_workertype()
                    if len(self._route) == 0: return None
                    next_lat = self._route[self._current_index_of_route]['lat']
                    next_lng = self._route[self._current_index_of_route]['lng']
                    self._manager_mutex.release()
                    return Location(next_lat, next_lng)
                self._manager_mutex.release()
                return self.get_next_location()
            self._last_round_prio = False
        log.info(
            "%s done grabbing next coord, releasing lock and returning location: %s, %s"
            % (str(self.name), str(next_lat), str(next_lng)))
        self._manager_mutex.release()
        if self._check_coords_before_returning(next_lat, next_lng):
            return Location(next_lat, next_lng)
        else:
            return self.get_next_location()

    def del_from_route(self):
        log.debug(
            "%s: Location available, acquiring lock and trying to return location"
            % str(self.name))
        self._manager_mutex.acquire()
        log.info('Removing coords from Route')
        self._route.pop(int(self._current_index_of_route) - 1)
        self._current_index_of_route -= 1
        if len(self._route) == 0:
            log.info('No more coords are available... Sleeping.')
        self._manager_mutex.release()

    def change_init_mapping(self, name_area):
        with open('configs/mappings.json') as f:
            vars = json.load(f)

        for var in vars['areas']:
            if (var['name']) == name_area:
                var['init'] = bool(False)

        with open('mappings.json', 'w') as outfile:
            json.dump(vars, outfile, indent=4, sort_keys=True)

    def get_route_status(self):
        if self._route:
            return self._current_index_of_route, len(self._route)
        return self._current_index_of_route, self._current_index_of_route