def stops_from_db_unvisited(self, geofence_helper: GeofenceHelper, origin: str): logger.debug("DbWrapper::stops_from_db_unvisited called") minLat, minLon, maxLat, maxLon = geofence_helper.get_polygon_from_fence( ) query = ( "SELECT pokestop.latitude, pokestop.longitude,GROUP_CONCAT(trs_visited.origin) AS vis " "FROM pokestop " "LEFT JOIN trs_visited ON pokestop.pokestop_id = trs_visited.pokestop_id " "WHERE (pokestop.latitude >= {} AND pokestop.longitude >= {} " "AND pokestop.latitude <= {} AND pokestop.longitude <= {}) " "GROUP by pokestop.pokestop_id HAVING INSTR(vis,'{}') < 1").format( minLat, minLon, maxLat, maxLon, origin) res = self.execute(query) unvisited: List[Location] = [] for (latitude, longitude, vis) in res: unvisited.append(Location(latitude, longitude)) if geofence_helper is not None: geofenced_coords = geofence_helper.get_geofenced_coordinates( unvisited) return geofenced_coords else: return unvisited
def __build_geofence_helpers(self, routemanagers): self.__geofence_helpers = [] if self.__args.webhook_excluded_areas == "": pass area_names = self.__args.webhook_excluded_areas.split(",") for area_name in area_names: if area_name.endswith("*"): for name, rmgr in routemanagers.items(): if not name.startswith(area_name[:-1]): continue self.__geofence_helpers.append( GeofenceHelper(rmgr["geofence_included"], rmgr["geofence_excluded"])) else: area = routemanagers.get(area_name, None) if area is None: continue self.__geofence_helpers.append( GeofenceHelper(area["geofence_included"], area["geofence_excluded"]))
def __init__(self, db_wrapper, coords, max_radius, max_coords_within_radius, path_to_include_geofence, path_to_exclude_geofence, routefile, mode=None, init=False, name="unknown", settings=None): self.db_wrapper = db_wrapper self.init = init self.name = name self._coords_unstructured = coords self.geofence_helper = GeofenceHelper(path_to_include_geofence, path_to_exclude_geofence) self._routefile = routefile self._max_radius = max_radius self._max_coords_within_radius = max_coords_within_radius self.settings = settings self.mode = mode self._is_started = False # we want to store the workers using the routemanager self._workers_registered = [] self._workers_registered_mutex = Lock() self._last_round_prio = False self._manager_mutex = RLock() self._round_started_time = None if coords is not None: if init: fenced_coords = coords else: fenced_coords = self.geofence_helper.get_geofenced_coordinates( coords) self._route = getJsonRoute(fenced_coords, max_radius, max_coords_within_radius, routefile) else: self._route = None self._current_index_of_route = 0 self._init_mode_rounds = 0 if self.settings is not None: self.delay_after_timestamp_prio = self.settings.get( "delay_after_prio_event", None) self.starve_route = self.settings.get("starve_route", False) else: self.delay_after_timestamp_prio = None self.starve_route = False # initialize priority queue variables self._prio_queue = None self._update_prio_queue_thread = None self._stop_update_thread = Event()
def __init__(self, db_wrapper, coords, maxRadius, maxCoordsWithinRadius, pathToIncludeGeofence, pathToExcludeGeofence, routefile, coords_spawns_known=False, delayAfterHatch=None, init=False, mode=None, settings=None, name="unknown"): # TODO: handle mode=None... # first filter the coords with the geofence given self.init = init self.mode = mode self.name = name self.settings = settings self.coords_spawns_known = coords_spawns_known self.__coords_unstructured = coords self.__geofenceHelper = GeofenceHelper(pathToIncludeGeofence, pathToExcludeGeofence) self.__db_wrapper = db_wrapper self.__managerMutex = Lock() self.__lastRoundEggHatch = False # boolean to prevent starvation in a simple way... self.__routefile = routefile self.__max_radius = maxRadius self.__max_coords_within_radius = maxCoordsWithinRadius self.__round_started_time = None if coords is not None: if init: fencedCoords = coords else: fencedCoords = self.__geofenceHelper.get_geofenced_coordinates( coords) self.__route = getJsonRoute(fencedCoords, maxRadius, maxCoordsWithinRadius, routefile) else: self.__route = None self.__currentIndexOfRoute = 0 # heapq of hatched eggs self.__delayAfterHatch = delayAfterHatch if self.__delayAfterHatch is not None: self.__raidQueue = [] self.__stopUpdateThread = Event() self.__updateRaidQueueThread = Thread( name='raidQUpdate', target=self.__updatePriorityQueueLoop) self.__updateRaidQueueThread.daemon = False self.__updateRaidQueueThread.start() else: self.__raidQueue = None
def __init__(self, db_wrapper, coords, max_radius, max_coords_within_radius, path_to_include_geofence, path_to_exclude_geofence, routefile, mode=None, init=False, name="unknown", settings=None): self.db_wrapper = db_wrapper self.init = init self.name = name self._coords_unstructured = coords self.geofence_helper = GeofenceHelper( path_to_include_geofence, path_to_exclude_geofence) self._routefile = routefile self._max_radius = max_radius self._max_coords_within_radius = max_coords_within_radius self.settings = settings self.mode = mode self._last_round_prio = False self._manager_mutex = Lock() self._round_started_time = None if coords is not None: if init: fenced_coords = coords else: fenced_coords = self.geofence_helper.get_geofenced_coordinates( coords) self._route = getJsonRoute( fenced_coords, max_radius, max_coords_within_radius, routefile) else: self._route = None self._current_index_of_route = 0 self._init_mode_rounds = 0 if settings is not None: self.delay_after_timestamp_prio = settings.get( "delay_after_prio_event", None) if self.delay_after_timestamp_prio == 0: self.delay_after_timestamp_prio = None self.starve_route = settings.get("starve_route", False) else: self.delay_after_timestamp_prio = None self.starve_route = False if self.delay_after_timestamp_prio is not None or mode == "iv_mitm": self._prio_queue = [] if mode != "iv_mitm": self.clustering_helper = ClusteringHelper(self._max_radius, self._max_coords_within_radius, self._cluster_priority_queue_criteria()) self._stop_update_thread = Event() self._update_prio_queue_thread = Thread(name="prio_queue_update_" + name, target=self._update_priority_queue_loop) self._update_prio_queue_thread.daemon = True self._update_prio_queue_thread.start() else: self._prio_queue = None
def get_geofences(mapping_manager, data_manager, fence_type=None): areas = mapping_manager.get_areas() geofences = {} for area_id, area in areas.items(): geo_include = data_manager.get_resource( 'geofence', identifier=area["geofence_included"]) geo_exclude_id = area.get("geofence_excluded", None) geo_exclude = None if geo_exclude_id is not None: geo_exclude = data_manager.get_resource('geofence', identifier=geo_exclude_id) if fence_type is not None and area['mode'] != fence_type: continue area_geofences = GeofenceHelper(geo_include, geo_exclude) include = {} exclude = {} for fences in area_geofences.geofenced_areas: include[fences['name']] = [] for fence in fences['polygon']: include[fences['name']].append( [getCoordFloat(fence['lat']), getCoordFloat(fence['lon'])]) for fences in area_geofences.excluded_areas: exclude[fences['name']] = [] for fence in fences['polygon']: exclude[fences['name']].append( [getCoordFloat(fence['lat']), getCoordFloat(fence['lon'])]) geofences[area_id] = {'include': include, 'exclude': exclude} return geofences
def validate_custom(self): issues = {} try: geofence_helper = GeofenceHelper(self, None) except Exception as err: issues = { 'invalid': [('fence_data', 'Must be one coord set per line (float,float)')] } return issues
def get_routemanagers(self): from multiprocessing.pool import ThreadPool global mode_mapping # returns list of routemanagers with area IDs areas = {} area_arr = self.__raw_json["areas"] thread_pool = ThreadPool(processes=4) areas_procs = {} for area in area_arr: if area["geofence_included"] is None: raise RuntimeError("Cannot work without geofence_included") geofence_included = Path(area["geofence_included"]) if not geofence_included.is_file(): log.error("Geofence included file configured does not exist") sys.exit(1) geofence_excluded_raw_path = area.get("geofence_excluded", None) if geofence_excluded_raw_path is not None: geofence_excluded = Path(geofence_excluded_raw_path) if not geofence_excluded.is_file(): log.error("Geofence excluded specified but does not exist") sys.exit(1) area_dict = {"mode": area["mode"], "geofence_included": area["geofence_included"], "geofence_excluded": area.get("geofence_excluded", None), "routecalc": area["routecalc"]} # also build a routemanager for each area... # grab coords # first check if init is false or raids_ocr is set as mode, if so, grab the coords from DB # coords = np.loadtxt(area["coords"], delimiter=',') geofence_helper = GeofenceHelper( area["geofence_included"], area.get("geofence_excluded", None)) mode = area["mode"] # build routemanagers if mode == "raids_ocr" or mode == "raids_mitm": route_manager = RouteManagerRaids(self.db_wrapper, None, mode_mapping[area["mode"]]["range"], mode_mapping[area["mode"] ]["max_count"], area["geofence_included"], area.get( "geofence_excluded", None), area["routecalc"], mode=area["mode"], settings=area.get( "settings", None), init=area.get("init", False), name=area.get( "name", "unknown") ) elif mode == "mon_mitm": route_manager = RouteManagerMon(self.db_wrapper, None, mode_mapping[area["mode"]]["range"], mode_mapping[area["mode"] ]["max_count"], area["geofence_included"], area.get( "geofence_excluded", None), area["routecalc"], mode=area["mode"], coords_spawns_known=area.get( "coords_spawns_known", False), init=area.get("init", False), name=area.get( "name", "unknown"), settings=area.get( "settings", None) ) elif mode == "iv_mitm": route_manager = RouteManagerIV(self.db_wrapper, None, 0, 999999, area["geofence_included"], area.get( "geofence_excluded", None), area["routecalc"], name=area.get( "name", "unknown"), settings=area.get( "settings", None), mode=mode ) elif mode == "pokestops": route_manager = RouteManagerMon(self.db_wrapper, None, mode_mapping[area["mode"]]["range"], mode_mapping[area["mode"] ]["max_count"], area["geofence_included"], area.get( "geofence_excluded", None), area["routecalc"], mode=area["mode"], init=area.get("init", False), name=area.get( "name", "unknown"), settings=area.get( "settings", None) ) else: log.error("Invalid mode found in mapping parser.") sys.exit(1) if not mode == "iv_mitm": if mode == "raids_ocr" or area.get("init", False) is False: # grab data from DB depending on mode # TODO: move routemanagers to factory if mode == "raids_ocr" or mode == "raids_mitm": coords = self.db_wrapper.gyms_from_db(geofence_helper) elif mode == "mon_mitm": spawn_known = area.get("coords_spawns_known", False) if spawn_known: log.info("Reading known Spawnpoints from DB") coords = self.db_wrapper.get_detected_spawns( geofence_helper) else: log.info("Reading unknown Spawnpoints from DB") coords = self.db_wrapper.get_undetected_spawns( geofence_helper) elif mode == "pokestops": coords = self.db_wrapper.stops_from_db(geofence_helper) else: log.fatal("Mode not implemented yet: %s" % str(mode)) exit(1) else: # calculate all level N cells (mapping back from mapping above linked to mode) # coords = S2Helper.get_s2_cells_from_fence(geofence=geofence_helper, # cell_size=mode_mapping[mode]["s2_cell_level"]) coords = S2Helper._generate_locations(mode_mapping[area["mode"]]["range"], geofence_helper) route_manager.add_coords_list(coords) max_radius = mode_mapping[area["mode"]]["range"] max_count_in_radius = mode_mapping[area["mode"]]["max_count"] if not area.get("init", False): log.info("Calculating route for %s" % str(area.get("name", "unknown"))) proc = thread_pool.apply_async(route_manager.recalc_route, args=(max_radius, max_count_in_radius, 0, False)) areas_procs[area["name"]] = proc else: log.info("Init mode enabled and more than 400 coords in init. Going row-based for %s" % str(area.get("name", "unknown"))) # we are in init, let's write the init route to file to make it visible in madmin if area["routecalc"] is not None: routefile = area["routecalc"] if os.path.isfile(routefile + '.calc'): os.remove(routefile + '.calc') with open(routefile + '.calc', 'a') as f: for loc in coords: f.write(str(loc.lat) + ', ' + str(loc.lng) + '\n') # gotta feed the route to routemanager... TODO: without recalc... proc = thread_pool.apply_async(route_manager.recalc_route, args=(1, 99999999, 0, False)) areas_procs[area["name"]] = proc # log.error("Calculated route, appending another coord and recalculating") area_dict["routemanager"] = route_manager areas[area["name"]] = area_dict for area in areas_procs.keys(): to_be_checked = areas_procs[area] log.debug(to_be_checked) to_be_checked.get() thread_pool.close() thread_pool.join() return areas
class RouteManagerBase(ABC): def __init__(self, db_wrapper, coords, max_radius, max_coords_within_radius, path_to_include_geofence, path_to_exclude_geofence, routefile, mode=None, init=False, name="unknown", settings=None): self.db_wrapper = db_wrapper self.init = init self.name = name self._coords_unstructured = coords self.geofence_helper = GeofenceHelper(path_to_include_geofence, path_to_exclude_geofence) self._routefile = routefile self._max_radius = max_radius self._max_coords_within_radius = max_coords_within_radius self.settings = settings self.mode = mode self._is_started = False # we want to store the workers using the routemanager self._workers_registered = [] self._workers_registered_mutex = Lock() self._last_round_prio = False self._manager_mutex = RLock() self._round_started_time = None if coords is not None: if init: fenced_coords = coords else: fenced_coords = self.geofence_helper.get_geofenced_coordinates( coords) self._route = getJsonRoute(fenced_coords, max_radius, max_coords_within_radius, routefile) else: self._route = None self._current_index_of_route = 0 self._init_mode_rounds = 0 if self.settings is not None: self.delay_after_timestamp_prio = self.settings.get( "delay_after_prio_event", None) self.starve_route = self.settings.get("starve_route", False) else: self.delay_after_timestamp_prio = None self.starve_route = False # initialize priority queue variables self._prio_queue = None self._update_prio_queue_thread = None self._stop_update_thread = Event() def __del__(self): if self._update_prio_queue_thread is not None: self._stop_update_thread.set() self._update_prio_queue_thread.join() def clear_coords(self): self._manager_mutex.acquire() self._coords_unstructured = None self._manager_mutex.release() def register_worker(self, worker_name): self._workers_registered_mutex.acquire() try: if worker_name in self._workers_registered: log.info("Worker %s already registered to routemanager %s" % (str(worker_name), str(self.name))) return False else: log.info("Worker %s registering to routemanager %s" % (str(worker_name), str(self.name))) self._workers_registered.append(worker_name) return True finally: self._workers_registered_mutex.release() def unregister_worker(self, worker_name): self._workers_registered_mutex.acquire() try: if worker_name in self._workers_registered: log.info("Worker %s unregistering from routemanager %s" % (str(worker_name), str(self.name))) self._workers_registered.remove(worker_name) else: # TODO: handle differently? log.info( "Worker %s failed unregistering from routemanager %s since subscription was previously " "lifted" % (str(worker_name), str(self.name))) if len(self._workers_registered) == 0 and self._is_started: log.info( "Routemanager %s does not have any subscribing workers anymore, calling stop" % str(self.name)) self._quit_route() finally: self._workers_registered_mutex.release() def _check_started(self): return self._is_started def _start_priority_queue(self): if (self._update_prio_queue_thread is None and (self.delay_after_timestamp_prio is not None or self.mode == "iv_mitm") and not self.mode == "pokestops"): self._prio_queue = [] if self.mode not in ["iv_mitm", "pokestops"]: self.clustering_helper = ClusteringHelper( self._max_radius, self._max_coords_within_radius, self._cluster_priority_queue_criteria()) self._update_prio_queue_thread = Thread( name="prio_queue_update_" + self.name, target=self._update_priority_queue_loop) self._update_prio_queue_thread.daemon = False self._update_prio_queue_thread.start() # list_coords is a numpy array of arrays! def add_coords_numpy(self, list_coords): fenced_coords = self.geofence_helper.get_geofenced_coordinates( list_coords) self._manager_mutex.acquire() if self._coords_unstructured is None: self._coords_unstructured = fenced_coords else: self._coords_unstructured = np.concatenate( (self._coords_unstructured, fenced_coords)) self._manager_mutex.release() def add_coords_list(self, list_coords): to_be_appended = np.zeros(shape=(len(list_coords), 2)) for i in range(len(list_coords)): to_be_appended[i][0] = list_coords[i][0] to_be_appended[i][1] = list_coords[i][1] self.add_coords_numpy(to_be_appended) @staticmethod def calculate_new_route(coords, max_radius, max_coords_within_radius, routefile, delete_old_route, num_procs=0): if delete_old_route and os.path.exists(routefile + ".calc"): log.debug("Deleting routefile...") os.remove(routefile + ".calc") new_route = getJsonRoute(coords, max_radius, max_coords_within_radius, num_processes=num_procs, routefile=routefile) return new_route def recalc_route(self, max_radius, max_coords_within_radius, num_procs=1, delete_old_route=False, nofile=False): current_coords = self._coords_unstructured if nofile: routefile = None else: routefile = self._routefile new_route = RouteManagerBase.calculate_new_route( current_coords, max_radius, max_coords_within_radius, routefile, delete_old_route, num_procs) self._manager_mutex.acquire() self._route = new_route self._current_index_of_route = 0 self._manager_mutex.release() def _update_priority_queue_loop(self): if self._priority_queue_update_interval( ) is None or self._priority_queue_update_interval() == 0: return while not self._stop_update_thread.is_set(): # retrieve the latest hatches from DB # newQueue = self._db_wrapper.get_next_raid_hatches(self._delayAfterHatch, self._geofenceHelper) new_queue = self._retrieve_latest_priority_queue() self._merge_priority_queue(new_queue) time.sleep(self._priority_queue_update_interval()) def _merge_priority_queue(self, new_queue): if new_queue is not None: self._manager_mutex.acquire() merged = set(new_queue + self._prio_queue) merged = list(merged) merged = self._filter_priority_queue_internal(merged) heapq.heapify(merged) self._prio_queue = merged self._manager_mutex.release() log.info("New priorityqueue: %s" % merged) def date_diff_in_seconds(self, dt2, dt1): timedelta = dt2 - dt1 return timedelta.days * 24 * 3600 + timedelta.seconds def dhms_from_seconds(self, seconds): minutes, seconds = divmod(seconds, 60) hours, minutes = divmod(minutes, 60) # days, hours = divmod(hours, 24) return hours, minutes, seconds def _get_round_finished_string(self): round_finish_time = datetime.now() round_completed_in = ( "%d hours, %d minutes, %d seconds" % (self.dhms_from_seconds( self.date_diff_in_seconds(round_finish_time, self._round_started_time)))) return round_completed_in @abstractmethod def _retrieve_latest_priority_queue(self): """ Method that's supposed to return a plain list containing (timestamp, Location) of the next events of interest :return: """ pass @abstractmethod def _start_routemanager(self): """ Starts priority queue or whatever the implementations require :return: """ pass @abstractmethod def _quit_route(self): """ Killing the Route Thread :return: """ pass @abstractmethod def _get_coords_post_init(self): """ Return list of coords to be fetched and used for routecalc :return: """ pass @abstractmethod def _check_coords_before_returning(self, lat, lng): """ Return list of coords to be fetched and used for routecalc :return: """ pass @abstractmethod def _recalc_route_workertype(self): """ Return a new route for worker :return: """ pass @abstractmethod def _get_coords_after_finish_route(self): """ Return list of coords to be fetched after finish a route :return: """ pass @abstractmethod def _cluster_priority_queue_criteria(self): """ If you do not want to have any filtering, simply return 0, 0, otherwise simply return timedelta_seconds, distance :return: """ @abstractmethod def _priority_queue_update_interval(self): """ The time to sleep in between consecutive updates of the priority queue :return: """ def _filter_priority_queue_internal(self, latest): """ Filter through the internal priority queue and cluster events within the timedelta and distance returned by _cluster_priority_queue_criteria :return: """ # timedelta_seconds = self._cluster_priority_queue_criteria() if self.mode == "iv_mitm": # exclude IV prioQ to also pass encounterIDs since we do not pass additional information through when # clustering return latest delete_seconds_passed = 0 if self.settings is not None: delete_seconds_passed = self.settings.get( "remove_from_queue_backlog", 0) if delete_seconds_passed is not None: delete_before = time.time() - delete_seconds_passed else: delete_before = 0 latest = [ to_keep for to_keep in latest if not to_keep[0] < delete_before ] # TODO: sort latest by modified flag of event # merged = self._merge_queue(latest, self._max_radius, 2, timedelta_seconds) merged = self.clustering_helper.get_clustered(latest) return merged def get_next_location(self): log.debug("get_next_location of %s called" % str(self.name)) if not self._is_started: log.info("Starting routemanager %s in get_next_location" % str(self.name)) self._start_routemanager() next_lat, next_lng = 0, 0 # first check if a location is available, if not, block until we have one... got_location = False while not got_location: log.debug("%s: Checking if a location is available..." % str(self.name)) self._manager_mutex.acquire() got_location = (self._prio_queue is not None and len(self._prio_queue) > 0 or (self._route is not None and len(self._route) > 0)) self._manager_mutex.release() if not got_location: log.debug("%s: No location available yet" % str(self.name)) time.sleep(0.5) log.debug( "%s: Location available, acquiring lock and trying to return location" % str(self.name)) self._manager_mutex.acquire() # check priority queue for items of priority that are past our time... # if that is not the case, simply increase the index in route and return the location on route # determine whether we move to the next location or the prio queue top's item if (self.delay_after_timestamp_prio is not None and ((not self._last_round_prio or self.starve_route) and self._prio_queue and len(self._prio_queue) > 0 and self._prio_queue[0][0] < time.time())): log.debug("%s: Priority event" % str(self.name)) next_stop = heapq.heappop(self._prio_queue)[1] next_lat = next_stop.lat next_lng = next_stop.lng self._last_round_prio = True log.info( "Round of route %s is moving to %s, %s for a priority event" % (str(self.name), str(next_lat), str(next_lng))) else: log.debug("%s: Moving on with route" % str(self.name)) if self._current_index_of_route == 0: if self._round_started_time is not None: log.info( "Round of route %s reached the first spot again. It took %s" % (str(self.name), str(self._get_round_finished_string()))) self._round_started_time = datetime.now() if len(self._route) == 0: return None log.info("Round of route %s started at %s" % (str(self.name), str(self._round_started_time))) # continue as usual if self._current_index_of_route < len(self._route): log.info( "%s: Moving on with location %s [%s/%s]" % (str(self.name), self._route[self._current_index_of_route], self._current_index_of_route + 1, len(self._route))) next_lat = self._route[self._current_index_of_route]['lat'] next_lng = self._route[self._current_index_of_route]['lng'] self._current_index_of_route += 1 if self.init and self._current_index_of_route >= len(self._route): self._init_mode_rounds += 1 if self.init and self._current_index_of_route >= len(self._route) and \ self._init_mode_rounds >= int(self.settings.get("init_mode_rounds", 1)): # we are done with init, let's calculate a new route log.warning( "Init of %s done, it took %s, calculating new route..." % (str(self.name), self._get_round_finished_string())) self.clear_coords() coords = self._get_coords_post_init() log.debug("Setting %s coords to as new points in route of %s" % (str(len(coords)), str(self.name))) self.add_coords_list(coords) log.debug("Route of %s is being calculated" % str(self.name)) self._recalc_route_workertype() self.init = False self.change_init_mapping(self.name) self._manager_mutex.release() return self.get_next_location() elif self._current_index_of_route == len(self._route): log.info('Reaching last coord of route') elif self._current_index_of_route > len(self._route): self._current_index_of_route = 0 coords_after_round = self._get_coords_after_finish_route() # TODO: check isinstance list? if coords_after_round is not None: self.clear_coords() coords = coords_after_round self.add_coords_list(coords) self._recalc_route_workertype() if len(self._route) == 0: return None next_lat = self._route[self._current_index_of_route]['lat'] next_lng = self._route[self._current_index_of_route]['lng'] self._manager_mutex.release() return Location(next_lat, next_lng) self._manager_mutex.release() return self.get_next_location() self._last_round_prio = False log.info( "%s done grabbing next coord, releasing lock and returning location: %s, %s" % (str(self.name), str(next_lat), str(next_lng))) self._manager_mutex.release() if self._check_coords_before_returning(next_lat, next_lng): return Location(next_lat, next_lng) else: return self.get_next_location() def del_from_route(self): log.debug( "%s: Location available, acquiring lock and trying to return location" % str(self.name)) self._manager_mutex.acquire() log.info('Removing coords from Route') self._route.pop(int(self._current_index_of_route) - 1) self._current_index_of_route -= 1 if len(self._route) == 0: log.info('No more coords are available... Sleeping.') self._manager_mutex.release() def change_init_mapping(self, name_area): with open('configs/mappings.json') as f: vars = json.load(f) for var in vars['areas']: if (var['name']) == name_area: var['init'] = bool(False) with open('mappings.json', 'w') as outfile: json.dump(vars, outfile, indent=4, sort_keys=True) def get_route_status(self): if self._route: return self._current_index_of_route, len(self._route) return self._current_index_of_route, self._current_index_of_route
def __init__(self, db_wrapper: DbWrapperBase, dbm: DataManager, area_id: str, coords: List[Location], max_radius: float, max_coords_within_radius: int, path_to_include_geofence: str, path_to_exclude_geofence: str, routefile: str, mode=None, init: bool = False, name: str = "unknown", settings: dict = None, level: bool = False, calctype: str = "optimized", useS2: bool = False, S2level: int = 15, joinqueue = None): self.db_wrapper: DbWrapperBase = db_wrapper self.init: bool = init self.name: str = name self._data_manager = dbm self.useS2: bool = useS2 self.S2level: int = S2level self.area_id = area_id self._coords_unstructured: List[Location] = coords self.geofence_helper: GeofenceHelper = GeofenceHelper( path_to_include_geofence, path_to_exclude_geofence) self._routefile = os.path.join(args.file_path, routefile) self._max_radius: float = max_radius self._max_coords_within_radius: int = max_coords_within_radius self.settings: dict = settings self.mode = mode self._is_started: bool = False self._first_started = False self._current_route_round_coords: List[Location] = [] self._start_calc: bool = False self._positiontyp = {} self._coords_to_be_ignored = set() self._level = level self._calctype = calctype self._overwrite_calculation: bool = False self._stops_not_processed: Dict[Location, int] = {} self._routepool: Dict[str, RoutePoolEntry] = {} self._roundcount: int = 0 self._joinqueue = joinqueue # we want to store the workers using the routemanager self._workers_registered: List[str] = [] self._workers_registered_mutex = RLock() # waiting till routepool is filled up self._workers_fillup_mutex = RLock() self._last_round_prio = {} self._manager_mutex = RLock() self._round_started_time = None self._route: List[Location] = [] if coords is not None: if init: fenced_coords = coords else: fenced_coords = self.geofence_helper.get_geofenced_coordinates( coords) new_coords = getJsonRoute( fenced_coords, max_radius, max_coords_within_radius, routefile, algorithm=calctype) for coord in new_coords: self._route.append(Location(coord["lat"], coord["lng"])) self._current_index_of_route = 0 self._init_mode_rounds = 0 if self.settings is not None: self.delay_after_timestamp_prio = self.settings.get( "delay_after_prio_event", None) self.starve_route = self.settings.get("starve_route", False) else: self.delay_after_timestamp_prio = None self.starve_route = False # initialize priority queue variables self._prio_queue = None self._update_prio_queue_thread = None self._check_routepools_thread = None self._stop_update_thread = Event()
def main(): args = Args() initLogging(args) if len(sys.argv) != 2: logger.error("usage: remove_all_spawns_within_geofence.py GEOFENCE_FILENAME") sys.exit(1) LocationWithID = collections.namedtuple('Location', ['lat', 'lng', 'spawnpoint']) geofence_filename = sys.argv[1] #print("Argument: '%s'" % (geofence_filename)) # no .txt, add it if ".txt" not in geofence_filename: geofence_filename = geofence_filename+".txt" # no / in filename, probably not an absolute path, append standard MAD path if "/" not in geofence_filename: geofence_filename = "../configs/geofences/" + geofence_filename logger.info("Trying to use file: {}", geofence_filename) if not os.path.isfile(geofence_filename): logger.error("Geofence file {} not found, exit", geofence_filename) sys.exit(1) geofence_helper = GeofenceHelper(geofence_filename, None) minLat, minLon, maxLat, maxLon = geofence_helper.get_polygon_from_fence() query = ( "SELECT latitude, longitude, spawnpoint " "FROM trs_spawn " "WHERE (latitude >= {} AND longitude >= {} " "AND latitude <= {} AND longitude <= {}) " ).format(minLat, minLon, maxLat, maxLon) delete_query = ( "DELETE FROM trs_spawn " "WHERE spawnpoint = {} " ) list_of_coords: List[LocationWithID] = [] dbip = get_value_for(r'\s+dbip:\s+([^\s]+)') dbport = get_value_for(r'\s+dbport:\s+([^.\s]*)', False) if dbport is None: # if dbport is not set, use default dbport = '3306' dbusername = get_value_for(r'\s+dbusername:\s+([^.\s]*)') dbpassword = get_value_for(r'\s+dbpassword:\s+([^.\s]*)') dbname = get_value_for(r'\s+dbname:\s+([^.\s]*)') #print("Successfully parsed config.ini, using values:") #print("dbport: %s" % dbport) #print("dbusername: %s" % dbusername) #print("dbname: %s" % dbname) #print("dbip: %s" % dbip) connection = mysql.connector.connect( host = dbip, port = dbport, user = dbusername, passwd = dbpassword, database = dbname) cursor = connection.cursor() cursor.execute(query) res = cursor.fetchall() for (latitude, longitude, spawnpoint) in res: list_of_coords.append(LocationWithID(latitude, longitude, spawnpoint)) geofenced_coords = geofence_helper.get_geofenced_coordinates(list_of_coords) spawnpointcount = len(geofenced_coords) for coords in geofenced_coords: sql = delete_query.format(coords.spawnpoint) cursor.execute(sql) #print(sql) connection.commit() cursor.close() connection.close() logger.success("Done, deleted {} spawnpoints", spawnpointcount)
class RouteManagerBase(ABC): def __init__(self, db_wrapper, coords, max_radius, max_coords_within_radius, path_to_include_geofence, path_to_exclude_geofence, routefile, init=False, name="unknown", settings=None): self.db_wrapper = db_wrapper self.init = init self.name = name self._coords_unstructured = coords self.geofence_helper = GeofenceHelper(path_to_include_geofence, path_to_exclude_geofence) self._routefile = routefile self._max_radius = max_radius self._max_coords_within_radius = max_coords_within_radius self.settings = settings self._last_round_prio = False self._manager_mutex = Lock() self._round_started_time = None if coords is not None: if init: fenced_coords = coords else: fenced_coords = self.geofence_helper.get_geofenced_coordinates(coords) self._route = getJsonRoute(fenced_coords, max_radius, max_coords_within_radius, routefile) else: self._route = None self._current_index_of_route = 0 if settings is not None: self.delay_after_timestamp_prio = settings.get("delay_after_prio_event", None) self.starve_route = settings.get("starve_route", False) else: self.delay_after_timestamp_prio = None self.starve_route = False if self.delay_after_timestamp_prio is not None: self._prio_queue = [] self.clustering_helper = ClusteringHelper(self._max_radius, self._max_coords_within_radius, self._cluster_priority_queue_criteria()) self._stop_update_thread = Event() self._update_prio_queue_thread = Thread(name="prio_queue_update_" + name, target=self._update_priority_queue_loop) self._update_prio_queue_thread.daemon = True self._update_prio_queue_thread.start() else: self._prio_queue = None def __del__(self): if self.delay_after_timestamp_prio: self._stop_update_thread.set() self._update_prio_queue_thread.join() def clear_coords(self): self._manager_mutex.acquire() self._coords_unstructured = None self._manager_mutex.release() # list_coords is a numpy array of arrays! def add_coords_numpy(self, list_coords): fenced_coords = self.geofence_helper.get_geofenced_coordinates(list_coords) self._manager_mutex.acquire() if self._coords_unstructured is None: self._coords_unstructured = fenced_coords else: self._coords_unstructured = np.concatenate((self._coords_unstructured, fenced_coords)) self._manager_mutex.release() def add_coords_list(self, list_coords): to_be_appended = np.zeros(shape=(len(list_coords), 2)) for i in range(len(list_coords)): to_be_appended[i][0] = list_coords[i][0] to_be_appended[i][1] = list_coords[i][1] self.add_coords_numpy(to_be_appended) @staticmethod def calculate_new_route(coords, max_radius, max_coords_within_radius, routefile, delete_old_route, num_procs=0): if delete_old_route and os.path.exists(routefile + ".calc"): log.debug("Deleting routefile...") os.remove(routefile + ".calc") new_route = getJsonRoute(coords, max_radius, max_coords_within_radius, num_processes=num_procs, routefile=routefile) return new_route def recalc_route(self, max_radius, max_coords_within_radius, num_procs=1, delete_old_route=False): current_coords = self._coords_unstructured routefile = self._routefile new_route = RouteManagerBase.calculate_new_route(current_coords, max_radius, max_coords_within_radius, routefile, delete_old_route, num_procs) self._manager_mutex.acquire() self._route = new_route self._current_index_of_route = 0 self._manager_mutex.release() def _update_priority_queue_loop(self): while not self._stop_update_thread.is_set(): # retrieve the latest hatches from DB # newQueue = self._db_wrapper.get_next_raid_hatches(self._delayAfterHatch, self._geofenceHelper) new_queue = self._retrieve_latest_priority_queue() self._merge_priority_queue(new_queue) time.sleep(300) def _merge_priority_queue(self, new_queue): self._manager_mutex.acquire() merged = list(set(new_queue + self._prio_queue)) merged = self._filter_priority_queue_internal(merged) heapq.heapify(merged) self._prio_queue = merged self._manager_mutex.release() log.info("New priorityqueue: %s" % merged) def date_diff_in_seconds(self, dt2, dt1): timedelta = dt2 - dt1 return timedelta.days * 24 * 3600 + timedelta.seconds def dhms_from_seconds(self, seconds): minutes, seconds = divmod(seconds, 60) hours, minutes = divmod(minutes, 60) # days, hours = divmod(hours, 24) return hours, minutes, seconds def _get_round_finished_string(self): round_finish_time = datetime.now() round_completed_in = ( "%d hours, %d minutes, %d seconds" % ( self.dhms_from_seconds( self.date_diff_in_seconds(round_finish_time, self._round_started_time) ) ) ) return round_completed_in @abstractmethod def _retrieve_latest_priority_queue(self): """ Method that's supposed to return a plain list containing (timestamp, Location) of the next events of interest :return: """ pass @abstractmethod def _get_coords_post_init(self): """ Return list of coords to be fetched and used for routecalc :return: """ pass @abstractmethod def _cluster_priority_queue_criteria(self): """ If you do not want to have any filtering, simply return 0, 0, otherwise simply return timedelta_seconds, distance :return: """ def _filter_priority_queue_internal(self, latest): """ Filter through the internal priority queue and cluster events within the timedelta and distance returned by _cluster_priority_queue_criteria :return: """ # timedelta_seconds = self._cluster_priority_queue_criteria() delete_seconds_passed = 0 if self.settings is not None: delete_seconds_passed = self.settings.get("remove_from_queue_backlog", 0) if delete_seconds_passed is not None: delete_before = time.time() - delete_seconds_passed else: delete_before = 0 latest = [to_keep for to_keep in latest if not to_keep[0] < delete_before] # TODO: sort latest by modified flag of event # merged = self._merge_queue(latest, self._max_radius, 2, timedelta_seconds) merged = self.clustering_helper.get_clustered(latest) return merged def get_next_location(self): log.debug("get_next_location of %s called" % str(self.name)) next_lat, next_lng = 0, 0 # first check if a location is available, if not, block until we have one... got_location = False while not got_location: log.debug("%s: Checking if a location is available..." % str(self.name)) self._manager_mutex.acquire() got_location = self._prio_queue is not None and len(self._prio_queue) > 0 or len(self._route) > 0 self._manager_mutex.release() if not got_location: log.debug("%s: No location available yet" % str(self.name)) time.sleep(0.5) log.debug("%s: Location available, acquiring lock and trying to return location") self._manager_mutex.acquire() # check priority queue for items of priority that are past our time... # if that is not the case, simply increase the index in route and return the location on route # determine whether we move to the next location or the prio queue top's item if (self.delay_after_timestamp_prio is not None and ((not self._last_round_prio or self.starve_route) and len(self._prio_queue) > 0 and self._prio_queue[0][0] < time.time())): log.debug("%s: Priority event" % str(self.name)) next_stop = heapq.heappop(self._prio_queue)[1] next_lat = next_stop.lat next_lng = next_stop.lng self._last_round_prio = True log.info("Round of route %s is moving to %s, %s for a priority event" % (str(self.name), str(next_lat), str(next_lng))) else: log.debug("%s: Moving on with route" % str(self.name)) if self._current_index_of_route == 0: if self._round_started_time is not None: log.info("Round of route %s reached the first spot again. It took %s" % (str(self.name), str(self._get_round_finished_string()))) self._round_started_time = datetime.now() log.info("Round of route %s started at %s" % (str(self.name), str(self._round_started_time))) # continue as usual log.info("Moving on with location %s" % self._route[self._current_index_of_route]) next_lat = self._route[self._current_index_of_route]['lat'] next_lng = self._route[self._current_index_of_route]['lng'] self._current_index_of_route += 1 if self.init and self._current_index_of_route >= len(self._route): # we are done with init, let's calculate a new route log.warning("Init of %s done, it took %s, calculating new route..." % (str(self.name), self._get_round_finished_string())) self._manager_mutex.release() self.clear_coords() coords = self._get_coords_post_init() log.debug("Setting %s coords to as new points in route of %s" % (str(len(coords)), str(self.name))) self.add_coords_list(coords) log.debug("Route of %s is being calculated" % str(self.name)) self.recalc_route(self._max_radius, self._max_coords_within_radius, 1, True) self.init = False return self.get_next_location() elif self._current_index_of_route >= len(self._route): self._current_index_of_route = 0 self._last_round_prio = False log.info("%s done grabbing next coord, releasing lock and returning location: %s, %s" % (str(self.name), str(next_lat), str(next_lng))) self._manager_mutex.release() return Location(next_lat, next_lng)
def __get_latest_routemanagers(self) -> Optional[Dict[str, dict]]: global mode_mapping areas: Optional[Dict[str, dict]] = {} if self.__configmode: return areas raw_areas = self.__data_manager.get_root_resource('area') thread_pool = ThreadPool(processes=4) areas_procs = {} for area_id, area_true in raw_areas.items(): area = area_true.get_resource() if area["geofence_included"] is None: raise RuntimeError("Cannot work without geofence_included") try: geofence_included = self.__data_manager.get_resource( 'geofence', identifier=area["geofence_included"]) except: raise RuntimeError( "geofence_included for area '{}' is specified but does not exist ('{}')." .format(area["name"], geofence_included)) geofence_excluded_raw_path = area.get("geofence_excluded", None) try: if geofence_excluded_raw_path is not None: geofence_excluded = self.__data_manager.get_resource( 'geofence', identifier=geofence_excluded_raw_path) else: geofence_excluded = None except: raise RuntimeError( "geofence_excluded for area '{}' is specified but file does not exist ('{}')." .format(area["name"], geofence_excluded_raw_path)) area_dict = { "mode": area_true.area_type, "geofence_included": geofence_included, "geofence_excluded": geofence_excluded, "routecalc": area["routecalc"], "name": area['name'] } # also build a routemanager for each area... # grab coords # first check if init is false, if so, grab the coords from DB # coords = np.loadtxt(area["coords"], delimiter=',') geofence_helper = GeofenceHelper(geofence_included, geofence_excluded) mode = area_true.area_type # build routemanagers # map iv list to ids if area.get('settings', None) is not None and 'mon_ids_iv' in area['settings']: # replace list name area['settings']['mon_ids_iv_raw'] = \ self.get_monlist(area['settings'].get('mon_ids_iv', None), area.get("name", "unknown")) route_resource = self.__data_manager.get_resource( 'routecalc', identifier=area["routecalc"]) route_manager = RouteManagerFactory.get_routemanager( self.__db_wrapper, self.__data_manager, area_id, None, mode_mapping.get(mode, {}).get("range", 0), mode_mapping.get(mode, {}).get("max_count", 99999999), geofence_included, path_to_exclude_geofence=geofence_excluded, mode=mode, settings=area.get("settings", None), init=area.get("init", False), name=area.get("name", "unknown"), level=area.get("level", False), coords_spawns_known=area.get("coords_spawns_known", False), routefile=route_resource, calctype=area.get("route_calc_algorithm", "optimized"), joinqueue=self.join_routes_queue, S2level=mode_mapping.get(mode, {}).get("s2_cell_level", 30)) if mode not in ("iv_mitm", "idle"): coords = self.__fetch_coords( mode, geofence_helper, coords_spawns_known=area.get("coords_spawns_known", False), init=area.get("init", False), range_init=mode_mapping.get(area_true.area_type, {}).get("range_init", 630), including_stops=area.get("including_stops", False)) route_manager.add_coords_list(coords) max_radius = mode_mapping[area_true.area_type]["range"] max_count_in_radius = mode_mapping[ area_true.area_type]["max_count"] if not area.get("init", False): logger.info("Initializing area {}", area["name"]) proc = thread_pool.apply_async( route_manager.initial_calculation, args=(max_radius, max_count_in_radius, 0, False)) areas_procs[area_id] = proc else: logger.info("Init mode enabled. Going row-based for {}", str(area.get("name", "unknown"))) # we are in init, let's write the init route to file to make it visible in madmin calc_coords = [] if area["routecalc"] is not None: for loc in coords: calc_coord = '%s,%s' % (str(loc.lat), str(loc.lng)) calc_coords.append(calc_coord) route_resource['routefile'] = calc_coords route_resource.save() # gotta feed the route to routemanager... TODO: without recalc... proc = thread_pool.apply_async(route_manager.recalc_route, args=(1, 99999999, 0, False)) areas_procs[area_id] = proc area_dict["routemanager"] = route_manager areas[area_id] = area_dict for area in areas_procs.keys(): to_be_checked = areas_procs[area] to_be_checked.get() thread_pool.close() thread_pool.join() return areas
def __init__(self, db_wrapper: DbWrapperBase, coords: List[Location], max_radius: float, max_coords_within_radius: int, path_to_include_geofence: str, path_to_exclude_geofence: str, routefile: str, mode=None, init: bool = False, name: str = "unknown", settings: dict = None): self.db_wrapper: DbWrapperBase = db_wrapper self.init: bool = init self.name: str = name self._coords_unstructured: List[Location] = coords self.geofence_helper: GeofenceHelper = GeofenceHelper( path_to_include_geofence, path_to_exclude_geofence) self._routefile = os.path.join(args.file_path, routefile) self._max_radius: float = max_radius self._max_coords_within_radius: int = max_coords_within_radius self.settings: dict = settings self.mode = mode self._is_started: bool = False self._first_started = False self._route_queue = Queue() self._start_calc: bool = False self._rounds = {} self._positiontyp = {} self._coords_to_be_ignored = set() # we want to store the workers using the routemanager self._workers_registered: List[WorkerBase] = [] self._workers_registered_mutex = Lock() self._last_round_prio = {} self._manager_mutex = RLock() self._round_started_time = None self._route: List[Location] = [] if coords is not None: if init: fenced_coords = coords else: fenced_coords = self.geofence_helper.get_geofenced_coordinates( coords) new_coords = getJsonRoute(fenced_coords, max_radius, max_coords_within_radius, routefile) for coord in new_coords: self._route.append(Location(coord["lat"], coord["lng"])) self._current_index_of_route = 0 self._init_mode_rounds = 0 if self.settings is not None: self.delay_after_timestamp_prio = self.settings.get( "delay_after_prio_event", None) self.starve_route = self.settings.get("starve_route", False) else: self.delay_after_timestamp_prio = None self.starve_route = False # initialize priority queue variables self._prio_queue = None self._update_prio_queue_thread = None self._stop_update_thread = Event()
class RouteManager: def __init__(self, db_wrapper, coords, maxRadius, maxCoordsWithinRadius, pathToIncludeGeofence, pathToExcludeGeofence, routefile, coords_spawns_known=False, delayAfterHatch=None, init=False, mode=None, settings=None, name="unknown"): # TODO: handle mode=None... # first filter the coords with the geofence given self.init = init self.mode = mode self.name = name self.settings = settings self.coords_spawns_known = coords_spawns_known self.__coords_unstructured = coords self.__geofenceHelper = GeofenceHelper(pathToIncludeGeofence, pathToExcludeGeofence) self.__db_wrapper = db_wrapper self.__managerMutex = Lock() self.__lastRoundEggHatch = False # boolean to prevent starvation in a simple way... self.__routefile = routefile self.__max_radius = maxRadius self.__max_coords_within_radius = maxCoordsWithinRadius self.__round_started_time = None if coords is not None: if init: fencedCoords = coords else: fencedCoords = self.__geofenceHelper.get_geofenced_coordinates( coords) self.__route = getJsonRoute(fencedCoords, maxRadius, maxCoordsWithinRadius, routefile) else: self.__route = None self.__currentIndexOfRoute = 0 # heapq of hatched eggs self.__delayAfterHatch = delayAfterHatch if self.__delayAfterHatch is not None: self.__raidQueue = [] self.__stopUpdateThread = Event() self.__updateRaidQueueThread = Thread( name='raidQUpdate', target=self.__updatePriorityQueueLoop) self.__updateRaidQueueThread.daemon = False self.__updateRaidQueueThread.start() else: self.__raidQueue = None def __del__(self): if self.__delayAfterHatch: self.__stopUpdateThread.set() self.__updateRaidQueueThread.join() def clear_coords(self): self.__managerMutex.acquire() self.__coords_unstructured = None self.__managerMutex.release() # list_coords is a numpy array of arrays! def add_coords_numpy(self, list_coords): fenced_coords = self.__geofenceHelper.get_geofenced_coordinates( list_coords) self.__managerMutex.acquire() if self.__coords_unstructured is None: self.__coords_unstructured = fenced_coords else: self.__coords_unstructured = np.concatenate( (self.__coords_unstructured, fenced_coords)) self.__managerMutex.release() def add_coords_list(self, list_coords): to_be_appended = np.zeros(shape=(len(list_coords), 2)) for i in range(len(list_coords)): to_be_appended[i][0] = list_coords[i][0] to_be_appended[i][1] = list_coords[i][1] self.add_coords_numpy(to_be_appended) @staticmethod def calculate_new_route(coords, max_radius, max_coords_within_radius, routefile, delete_old_route, num_procs=0): if delete_old_route and os.path.exists(routefile + ".calc"): log.debug("Deleting routefile...") os.remove(routefile + ".calc") new_route = getJsonRoute(coords, max_radius, max_coords_within_radius, num_processes=num_procs, routefile=routefile) return new_route def recalc_route(self, max_radius, max_coords_within_radius, num_procs=1, delete_old_route=False): current_coords = self.__coords_unstructured routefile = self.__routefile new_route = RouteManager.calculate_new_route( current_coords, max_radius, max_coords_within_radius, routefile, delete_old_route, num_procs) self.__managerMutex.acquire() self.__route = new_route self.__currentIndexOfRoute = 0 self.__managerMutex.release() def __updatePriorityQueueLoop(self): while not self.__stopUpdateThread.is_set(): # retrieve the latest hatches from DB newQueue = self.__db_wrapper.get_next_raid_hatches( self.__delayAfterHatch, self.__geofenceHelper) self.__mergeRaidQueue(newQueue) time.sleep(300) def __mergeRaidQueue(self, newQueue): self.__managerMutex.acquire() merged = list(set(newQueue + self.__raidQueue)) heapq.heapify(merged) self.__raidQueue = merged self.__managerMutex.release() log.info("New raidqueue: %s" % merged) def getNextLocation(self): # nextLocation = Location(None, None) nextLat = 0 nextLng = 0 # first check if a location is available, if not, block until we have one... TODO got_location = False while not got_location: self.__managerMutex.acquire() got_location = self.__raidQueue is not None and len( self.__raidQueue) > 0 or len(self.__route) > 0 self.__managerMutex.release() if not got_location: time.sleep(0.5) self.__managerMutex.acquire() # check raid queue for hatches, if none have passed, simply increase index and return the location in the route # at index # determine whether we move to the next gym or to the top of our priority queue if self.__delayAfterHatch is not None and ( not self.__lastRoundEggHatch and len(self.__raidQueue) > 0 and self.__raidQueue[0][0] < time.time()): nextStop = heapq.heappop( self.__raidQueue)[1] # gets the location tuple nextLat = nextStop.latitude nextLng = nextStop.longitude self.__lastRoundEggHatch = True else: if self.__currentIndexOfRoute == 0: if self.__round_started_time is not None: log.info( "Round of route %s reached the first spot again. It took: %s" % (str(self.name), self.get_round_finished_string())) self.__round_started_time = datetime.now() log.info("Round of route %s started at %s" % (str(self.name), str(self.__round_started_time))) # continue as usual log.info('main: Moving on with gym at %s' % self.__route[self.__currentIndexOfRoute]) nextLat = self.__route[self.__currentIndexOfRoute]['lat'] nextLng = self.__route[self.__currentIndexOfRoute]['lng'] self.__currentIndexOfRoute += 1 if self.init and self.__currentIndexOfRoute >= len(self.__route): # we are done with init, let's calculate a new route log.warning( "Init of %s done, it took %s, calculating new route..." % (str(self.name), self.get_round_finished_string())) self.__managerMutex.release() self.clear_coords() if self.mode == "raids_ocr" or self.mode == "raids_mitm": coords = self.__db_wrapper.gyms_from_db( self.__geofenceHelper) elif self.mode == "mon_mitm": if self.coords_spawns_known: log.info("Reading known Spawnpoints from DB") coords = self.__db_wrapper.get_detected_spawns( self.__geofenceHelper) else: log.info("Reading unknown Spawnpoints from DB") coords = self.__db_wrapper.get_undetected_spawns( self.__geofenceHelper) else: log.fatal("Mode not implemented yet: %s" % str(self.mode)) exit(1) log.debug("Adding %s coords to list" % str(len(coords))) self.add_coords_list(coords) log.debug("Runnig calculation") self.recalc_route(self.__max_radius, self.__max_coords_within_radius, 1, True) self.init = False log.debug("Calling ourself to get coord...") return self.getNextLocation() elif self.__currentIndexOfRoute >= len(self.__route): self.__currentIndexOfRoute = 0 self.__lastRoundEggHatch = False self.__managerMutex.release() return Location(nextLat, nextLng) def date_diff_in_seconds(self, dt2, dt1): timedelta = dt2 - dt1 return timedelta.days * 24 * 3600 + timedelta.seconds def dhms_from_seconds(self, seconds): minutes, seconds = divmod(seconds, 60) hours, minutes = divmod(minutes, 60) # days, hours = divmod(hours, 24) return hours, minutes, seconds def get_round_finished_string(self): round_finish_time = datetime.now() round_completed_in = ( "%d hours, %d minutes, %d seconds" % (self.dhms_from_seconds( self.date_diff_in_seconds(round_finish_time, self.__round_started_time)))) return round_completed_in
def __get_latest_routemanagers(self) -> Optional[Dict[str, dict]]: global mode_mapping areas: Optional[Dict[str, dict]] = {} if self.__configmode: return areas area_arr = self.__raw_json["areas"] thread_pool = ThreadPool(processes=4) areas_procs = {} for area in area_arr: if area["geofence_included"] is None: raise RuntimeError("Cannot work without geofence_included") geofence_included = Path(area["geofence_included"]) if not geofence_included.is_file(): raise RuntimeError( "geofence_included for area '{}' is specified but file does not exist ('{}').".format( area["name"], geofence_included.resolve() ) ) geofence_excluded_raw_path = area.get("geofence_excluded", None) if geofence_excluded_raw_path is not None: geofence_excluded = Path(geofence_excluded_raw_path) if not geofence_excluded.is_file(): raise RuntimeError( "geofence_excluded for area '{}' is specified but file does not exist ('{}').".format( area["name"], geofence_excluded.resolve() ) ) area_dict = {"mode": area["mode"], "geofence_included": area["geofence_included"], "geofence_excluded": area.get("geofence_excluded", None), "routecalc": area["routecalc"]} # also build a routemanager for each area... # grab coords # first check if init is false, if so, grab the coords from DB # coords = np.loadtxt(area["coords"], delimiter=',') geofence_helper = GeofenceHelper( area["geofence_included"], area.get("geofence_excluded", None)) mode = area["mode"] # build routemanagers #map iv list to ids if area.get('settings', None) is not None and 'mon_ids_iv' in area['settings']: # replace list name area['settings']['mon_ids_iv_raw'] = \ self.get_monlist(area['settings'].get('mon_ids_iv', None), area.get("name", "unknown")) route_manager = RouteManagerFactory.get_routemanager(self.__db_wrapper, None, mode_mapping.get(mode, {}).get("range", 0), mode_mapping.get(mode, {}).get("max_count", 99999999), area["geofence_included"], area.get("geofence_excluded", None), mode=mode, settings=area.get("settings", None), init=area.get("init", False), name=area.get("name", "unknown"), level=area.get("level", False), coords_spawns_known=area.get( "coords_spawns_known", False), routefile=area["routecalc"], calctype=area.get("route_calc_algorithm", "optimized"), joinqueue=self.join_routes_queue ) if mode not in ("iv_mitm", "idle"): coords = self.__fetch_coords(mode, geofence_helper, coords_spawns_known=area.get("coords_spawns_known", False), init=area.get("init", False), range_init=mode_mapping.get(area["mode"], {}).get("range_init", 630), including_stops=area.get("including_stops", False)) route_manager.add_coords_list(coords) max_radius = mode_mapping[area["mode"]]["range"] max_count_in_radius = mode_mapping[area["mode"]]["max_count"] if not area.get("init", False): logger.info("Initializing area {}", area["name"]) proc = thread_pool.apply_async(route_manager.recalc_route, args=(max_radius, max_count_in_radius, 0, False)) areas_procs[area["name"]] = proc else: logger.info( "Init mode enabled. Going row-based for {}", str(area.get("name", "unknown"))) # we are in init, let's write the init route to file to make it visible in madmin if area["routecalc"] is not None: routefile = os.path.join( self.__args.file_path, area["routecalc"]) if os.path.isfile(routefile + '.calc'): os.remove(routefile + '.calc') with open(routefile + '.calc', 'a') as f: for loc in coords: f.write(str(loc.lat) + ', ' + str(loc.lng) + '\n') # gotta feed the route to routemanager... TODO: without recalc... proc = thread_pool.apply_async(route_manager.recalc_route, args=(1, 99999999, 0, False)) areas_procs[area["name"]] = proc area_dict["routemanager"] = route_manager areas[area["name"]] = area_dict for area in areas_procs.keys(): to_be_checked = areas_procs[area] to_be_checked.get() thread_pool.close() thread_pool.join() return areas
def _generate_locations(distance: float, geofence_helper: GeofenceHelper): results = [] south, east, north, west = geofence_helper.get_polygon_from_fence() corners = [ Location(south, east), Location(south, west), Location(north, east), Location(north, west) ] # get the center center = get_middle_of_coord_list(corners) # get the farthest to the center... farthest_dist = 0 for corner in corners: dist_temp = get_distance_of_two_points_in_meters( center.lat, center.lng, corner.lat, corner.lng) if dist_temp > farthest_dist: farthest_dist = dist_temp # calculate step_limit, round up to reduce risk of losing stuff step_limit = math.ceil(farthest_dist / distance) # This will loop thorugh all the rings in the hex from the centre # moving outwards logger.info("Calculating positions for init scan") num_cores = multiprocessing.cpu_count() with multiprocessing.Pool(processes=num_cores) as pool: temp = [ pool.apply(S2Helper._generate_star_locs, args=(center, distance, i)) for i in range(1, step_limit) ] results = [item for sublist in temp for item in sublist] results.append(Location(center.lat, center.lng)) # for ring in range(1, step_limit): # for i in range(0, 6): # # Star_locs will contain the locations of the 6 vertices of # # the current ring (90,150,210,270,330 and 30 degrees from # # origin) to form a star # star_loc = S2Helper.get_new_coords(center, distance * ring, # 90 + 60 * i) # for j in range(0, ring): # # Then from each point on the star, create locations # # towards the next point of star along the edge of the # # current ring # loc = S2Helper.get_new_coords(star_loc, distance * j, 210 + 60 * i) # results.append(loc) logger.info("Filtering positions for init scan") # Geofence results. if geofence_helper is not None and geofence_helper.is_enabled(): results = geofence_helper.get_geofenced_coordinates(results) if not results: logger.error( 'No cells regarded as valid for desired scan area. ' 'Check your provided geofences. Aborting.') sys.exit(1) logger.info("Ordering location") results = S2Helper.order_location_list_rows(results) return results
def get_routemanagers(self, delay_after_hatch=None): from multiprocessing.pool import ThreadPool global mode_mapping # returns list of routemanagers with area IDs areas = {} area_arr = self.__raw_json["areas"] thread_pool = ThreadPool(processes=4) areas_procs = {} for area in area_arr: if area["geofence_included"] is None: raise RuntimeError("Cannot work without geofence_included") geofence_included = Path(area["geofence_included"]) if not geofence_included.is_file(): log.error("Geofence included file configured does not exist") sys.exit(1) geofence_excluded_raw_path = area.get("geofence_excluded", None) if geofence_excluded_raw_path is not None: geofence_excluded = Path(geofence_excluded_raw_path) if not geofence_excluded.is_file(): log.error("Geofence excluded specified but does not exist") sys.exit(1) area_dict = {"mode": area["mode"], "geofence_included": area["geofence_included"], "geofence_excluded": area.get("geofence_excluded", None), "routecalc": area["routecalc"]} # also build a routemanager for each area... # grab coords # first check if init is false or raids_ocr is set as mode, if so, grab the coords from DB # coords = np.loadtxt(area["coords"], delimiter=',') geofence_helper = GeofenceHelper(area["geofence_included"], area.get("geofence_excluded", None)) mode = area["mode"] if mode == "raids_ocr" or area.get("init", False) is False: # grab data from DB depending on mode if mode == "raids_ocr" or mode == "raids_mitm": coords = self.db_wrapper.gyms_from_db(geofence_helper) elif mode == "mon_mitm": spawn_known = area.get("coords_spawns_known", False) if spawn_known: log.info("Reading known Spawnpoints from DB") coords = self.db_wrapper.get_detected_spawns(geofence_helper) else: log.info("Reading unknown Spawnpoints from DB") coords = self.db_wrapper.get_undetected_spawns(geofence_helper) else: log.fatal("Mode not implemented yet: %s" % str(mode)) exit(1) else: # calculate all level N cells (mapping back from mapping above linked to mode) coords = S2Helper.get_s2_cells_from_fence(geofence=geofence_helper, cell_size=mode_mapping[mode]["s2_cell_level"]) # coords = S2Helper._generate_locations(mode_mapping[area["mode"]]["range"], # geofence_helper) # retrieve the range and max count per circle from central mapping... route_manager = RouteManager(self.db_wrapper, None, mode_mapping[area["mode"]]["range"], mode_mapping[area["mode"]]["max_count"], area["geofence_included"], area.get("geofence_excluded", None), area["routecalc"], coords_spawns_known=area.get("coords_spawns_known", False), delayAfterHatch=delay_after_hatch, init=area.get("init", False), mode=area["mode"], settings=area["settings"], name=area.get("name", "unknwon")) route_manager.add_coords_list(coords) max_radius = mode_mapping[area["mode"]]["range"] max_count_in_radius = mode_mapping[area["mode"]]["max_count"] proc = thread_pool.apply_async(route_manager.recalc_route, args=(max_radius, max_count_in_radius, 0, False)) areas_procs[area["name"]] = proc # log.error("Calculated route, appending another coord and recalculating") area_dict["routemanager"] = route_manager areas[area["name"]] = area_dict for area in areas_procs.keys(): to_be_checked = areas_procs[area] log.debug(to_be_checked) to_be_checked.get() thread_pool.close() thread_pool.join() return areas
from pathlib import Path from geofence.geofenceHelper import GeofenceHelper import argparse log = logging.getLogger(__name__) parser = argparse.ArgumentParser(description="A program that checks if a lat/lon coord is contained within a geofence") parser.add_argument("-i", "--include", default="geofence_inc.txt", help="Geofence include file") parser.add_argument("-e", "--exclude", default=None, help="Geofence exclude file") parser.add_argument("-lat", "--latitude", type=float, required=True, help="Latitude of coordinate") parser.add_argument("-lon", "--longitude", type=float, required=True, help="Longitude of coordinate") args = parser.parse_args() if not Path(args.include).is_file(): raise RuntimeError("Geofence included file configured does not exist " + args.include) if (args.exclude != None ) and (not Path(args.exclude).is_file()): raise RuntimeError("Geofence excluded file configured does not exist " + args.exclude) geofence_helper = GeofenceHelper(args.include, args.exclude) if geofence_helper.is_coord_inside_include_geofence([args.latitude, args.longitude]): exit(0) else: exit(1)