def stops_from_db_unvisited(self, geofence_helper: GeofenceHelper,
origin: str):
logger.debug("DbWrapper::stops_from_db_unvisited called")
minLat, minLon, maxLat, maxLon = geofence_helper.get_polygon_from_fence(
)
query = (
"SELECT pokestop.latitude, pokestop.longitude,GROUP_CONCAT(trs_visited.origin) AS vis "
"FROM pokestop "
"LEFT JOIN trs_visited ON pokestop.pokestop_id = trs_visited.pokestop_id "
"WHERE (pokestop.latitude >= {} AND pokestop.longitude >= {} "
"AND pokestop.latitude <= {} AND pokestop.longitude <= {}) "
"GROUP by pokestop.pokestop_id HAVING INSTR(vis,'{}') < 1").format(
minLat, minLon, maxLat, maxLon, origin)
res = self.execute(query)
unvisited: List[Location] = []
for (latitude, longitude, vis) in res:
unvisited.append(Location(latitude, longitude))
if geofence_helper is not None:
geofenced_coords = geofence_helper.get_geofenced_coordinates(
unvisited)
return geofenced_coords
else:
return unvisited
def __build_geofence_helpers(self, routemanagers):
self.__geofence_helpers = []
if self.__args.webhook_excluded_areas == "":
pass
area_names = self.__args.webhook_excluded_areas.split(",")
for area_name in area_names:
if area_name.endswith("*"):
for name, rmgr in routemanagers.items():
if not name.startswith(area_name[:-1]):
continue
self.__geofence_helpers.append(
GeofenceHelper(rmgr["geofence_included"],
rmgr["geofence_excluded"]))
else:
area = routemanagers.get(area_name, None)
if area is None:
continue
self.__geofence_helpers.append(
GeofenceHelper(area["geofence_included"],
area["geofence_excluded"]))
def __init__(self,
db_wrapper,
coords,
max_radius,
max_coords_within_radius,
path_to_include_geofence,
path_to_exclude_geofence,
routefile,
mode=None,
init=False,
name="unknown",
settings=None):
self.db_wrapper = db_wrapper
self.init = init
self.name = name
self._coords_unstructured = coords
self.geofence_helper = GeofenceHelper(path_to_include_geofence,
path_to_exclude_geofence)
self._routefile = routefile
self._max_radius = max_radius
self._max_coords_within_radius = max_coords_within_radius
self.settings = settings
self.mode = mode
self._is_started = False
# we want to store the workers using the routemanager
self._workers_registered = []
self._workers_registered_mutex = Lock()
self._last_round_prio = False
self._manager_mutex = RLock()
self._round_started_time = None
if coords is not None:
if init:
fenced_coords = coords
else:
fenced_coords = self.geofence_helper.get_geofenced_coordinates(
coords)
self._route = getJsonRoute(fenced_coords, max_radius,
max_coords_within_radius, routefile)
else:
self._route = None
self._current_index_of_route = 0
self._init_mode_rounds = 0
if self.settings is not None:
self.delay_after_timestamp_prio = self.settings.get(
"delay_after_prio_event", None)
self.starve_route = self.settings.get("starve_route", False)
else:
self.delay_after_timestamp_prio = None
self.starve_route = False
# initialize priority queue variables
self._prio_queue = None
self._update_prio_queue_thread = None
self._stop_update_thread = Event()
def __init__(self,
db_wrapper,
coords,
maxRadius,
maxCoordsWithinRadius,
pathToIncludeGeofence,
pathToExcludeGeofence,
routefile,
coords_spawns_known=False,
delayAfterHatch=None,
init=False,
mode=None,
settings=None,
name="unknown"):
# TODO: handle mode=None...
# first filter the coords with the geofence given
self.init = init
self.mode = mode
self.name = name
self.settings = settings
self.coords_spawns_known = coords_spawns_known
self.__coords_unstructured = coords
self.__geofenceHelper = GeofenceHelper(pathToIncludeGeofence,
pathToExcludeGeofence)
self.__db_wrapper = db_wrapper
self.__managerMutex = Lock()
self.__lastRoundEggHatch = False # boolean to prevent starvation in a simple way...
self.__routefile = routefile
self.__max_radius = maxRadius
self.__max_coords_within_radius = maxCoordsWithinRadius
self.__round_started_time = None
if coords is not None:
if init:
fencedCoords = coords
else:
fencedCoords = self.__geofenceHelper.get_geofenced_coordinates(
coords)
self.__route = getJsonRoute(fencedCoords, maxRadius,
maxCoordsWithinRadius, routefile)
else:
self.__route = None
self.__currentIndexOfRoute = 0
# heapq of hatched eggs
self.__delayAfterHatch = delayAfterHatch
if self.__delayAfterHatch is not None:
self.__raidQueue = []
self.__stopUpdateThread = Event()
self.__updateRaidQueueThread = Thread(
name='raidQUpdate', target=self.__updatePriorityQueueLoop)
self.__updateRaidQueueThread.daemon = False
self.__updateRaidQueueThread.start()
else:
self.__raidQueue = None
def __init__(self, db_wrapper, coords, max_radius, max_coords_within_radius, path_to_include_geofence,
path_to_exclude_geofence, routefile, mode=None, init=False,
name="unknown", settings=None):
self.db_wrapper = db_wrapper
self.init = init
self.name = name
self._coords_unstructured = coords
self.geofence_helper = GeofenceHelper(
path_to_include_geofence, path_to_exclude_geofence)
self._routefile = routefile
self._max_radius = max_radius
self._max_coords_within_radius = max_coords_within_radius
self.settings = settings
self.mode = mode
self._last_round_prio = False
self._manager_mutex = Lock()
self._round_started_time = None
if coords is not None:
if init:
fenced_coords = coords
else:
fenced_coords = self.geofence_helper.get_geofenced_coordinates(
coords)
self._route = getJsonRoute(
fenced_coords, max_radius, max_coords_within_radius, routefile)
else:
self._route = None
self._current_index_of_route = 0
self._init_mode_rounds = 0
if settings is not None:
self.delay_after_timestamp_prio = settings.get(
"delay_after_prio_event", None)
if self.delay_after_timestamp_prio == 0:
self.delay_after_timestamp_prio = None
self.starve_route = settings.get("starve_route", False)
else:
self.delay_after_timestamp_prio = None
self.starve_route = False
if self.delay_after_timestamp_prio is not None or mode == "iv_mitm":
self._prio_queue = []
if mode != "iv_mitm":
self.clustering_helper = ClusteringHelper(self._max_radius,
self._max_coords_within_radius,
self._cluster_priority_queue_criteria())
self._stop_update_thread = Event()
self._update_prio_queue_thread = Thread(name="prio_queue_update_" + name,
target=self._update_priority_queue_loop)
self._update_prio_queue_thread.daemon = True
self._update_prio_queue_thread.start()
else:
self._prio_queue = None
def get_geofences(mapping_manager, data_manager, fence_type=None):
areas = mapping_manager.get_areas()
geofences = {}
for area_id, area in areas.items():
geo_include = data_manager.get_resource(
'geofence', identifier=area["geofence_included"])
geo_exclude_id = area.get("geofence_excluded", None)
geo_exclude = None
if geo_exclude_id is not None:
geo_exclude = data_manager.get_resource('geofence',
identifier=geo_exclude_id)
if fence_type is not None and area['mode'] != fence_type:
continue
area_geofences = GeofenceHelper(geo_include, geo_exclude)
include = {}
exclude = {}
for fences in area_geofences.geofenced_areas:
include[fences['name']] = []
for fence in fences['polygon']:
include[fences['name']].append(
[getCoordFloat(fence['lat']),
getCoordFloat(fence['lon'])])
for fences in area_geofences.excluded_areas:
exclude[fences['name']] = []
for fence in fences['polygon']:
exclude[fences['name']].append(
[getCoordFloat(fence['lat']),
getCoordFloat(fence['lon'])])
geofences[area_id] = {'include': include, 'exclude': exclude}
return geofences
def validate_custom(self):
issues = {}
try:
geofence_helper = GeofenceHelper(self, None)
except Exception as err:
issues = {
'invalid': [('fence_data',
'Must be one coord set per line (float,float)')]
}
return issues
def get_routemanagers(self):
from multiprocessing.pool import ThreadPool
global mode_mapping
# returns list of routemanagers with area IDs
areas = {}
area_arr = self.__raw_json["areas"]
thread_pool = ThreadPool(processes=4)
areas_procs = {}
for area in area_arr:
if area["geofence_included"] is None:
raise RuntimeError("Cannot work without geofence_included")
geofence_included = Path(area["geofence_included"])
if not geofence_included.is_file():
log.error("Geofence included file configured does not exist")
sys.exit(1)
geofence_excluded_raw_path = area.get("geofence_excluded", None)
if geofence_excluded_raw_path is not None:
geofence_excluded = Path(geofence_excluded_raw_path)
if not geofence_excluded.is_file():
log.error("Geofence excluded specified but does not exist")
sys.exit(1)
area_dict = {"mode": area["mode"],
"geofence_included": area["geofence_included"],
"geofence_excluded": area.get("geofence_excluded", None),
"routecalc": area["routecalc"]}
# also build a routemanager for each area...
# grab coords
# first check if init is false or raids_ocr is set as mode, if so, grab the coords from DB
# coords = np.loadtxt(area["coords"], delimiter=',')
geofence_helper = GeofenceHelper(
area["geofence_included"], area.get("geofence_excluded", None))
mode = area["mode"]
# build routemanagers
if mode == "raids_ocr" or mode == "raids_mitm":
route_manager = RouteManagerRaids(self.db_wrapper, None, mode_mapping[area["mode"]]["range"],
mode_mapping[area["mode"]
]["max_count"],
area["geofence_included"], area.get(
"geofence_excluded", None),
area["routecalc"],
mode=area["mode"], settings=area.get(
"settings", None),
init=area.get("init", False),
name=area.get(
"name", "unknown")
)
elif mode == "mon_mitm":
route_manager = RouteManagerMon(self.db_wrapper, None, mode_mapping[area["mode"]]["range"],
mode_mapping[area["mode"]
]["max_count"],
area["geofence_included"], area.get(
"geofence_excluded", None),
area["routecalc"], mode=area["mode"],
coords_spawns_known=area.get(
"coords_spawns_known", False),
init=area.get("init", False),
name=area.get(
"name", "unknown"),
settings=area.get(
"settings", None)
)
elif mode == "iv_mitm":
route_manager = RouteManagerIV(self.db_wrapper, None, 0, 999999,
area["geofence_included"], area.get(
"geofence_excluded", None),
area["routecalc"], name=area.get(
"name", "unknown"),
settings=area.get(
"settings", None),
mode=mode
)
elif mode == "pokestops":
route_manager = RouteManagerMon(self.db_wrapper, None, mode_mapping[area["mode"]]["range"],
mode_mapping[area["mode"]
]["max_count"],
area["geofence_included"], area.get(
"geofence_excluded", None),
area["routecalc"], mode=area["mode"],
init=area.get("init", False),
name=area.get(
"name", "unknown"),
settings=area.get(
"settings", None)
)
else:
log.error("Invalid mode found in mapping parser.")
sys.exit(1)
if not mode == "iv_mitm":
if mode == "raids_ocr" or area.get("init", False) is False:
# grab data from DB depending on mode
# TODO: move routemanagers to factory
if mode == "raids_ocr" or mode == "raids_mitm":
coords = self.db_wrapper.gyms_from_db(geofence_helper)
elif mode == "mon_mitm":
spawn_known = area.get("coords_spawns_known", False)
if spawn_known:
log.info("Reading known Spawnpoints from DB")
coords = self.db_wrapper.get_detected_spawns(
geofence_helper)
else:
log.info("Reading unknown Spawnpoints from DB")
coords = self.db_wrapper.get_undetected_spawns(
geofence_helper)
elif mode == "pokestops":
coords = self.db_wrapper.stops_from_db(geofence_helper)
else:
log.fatal("Mode not implemented yet: %s" % str(mode))
exit(1)
else:
# calculate all level N cells (mapping back from mapping above linked to mode)
# coords = S2Helper.get_s2_cells_from_fence(geofence=geofence_helper,
# cell_size=mode_mapping[mode]["s2_cell_level"])
coords = S2Helper._generate_locations(mode_mapping[area["mode"]]["range"],
geofence_helper)
route_manager.add_coords_list(coords)
max_radius = mode_mapping[area["mode"]]["range"]
max_count_in_radius = mode_mapping[area["mode"]]["max_count"]
if not area.get("init", False):
log.info("Calculating route for %s" %
str(area.get("name", "unknown")))
proc = thread_pool.apply_async(route_manager.recalc_route, args=(max_radius, max_count_in_radius,
0, False))
areas_procs[area["name"]] = proc
else:
log.info("Init mode enabled and more than 400 coords in init. Going row-based for %s"
% str(area.get("name", "unknown")))
# we are in init, let's write the init route to file to make it visible in madmin
if area["routecalc"] is not None:
routefile = area["routecalc"]
if os.path.isfile(routefile + '.calc'):
os.remove(routefile + '.calc')
with open(routefile + '.calc', 'a') as f:
for loc in coords:
f.write(str(loc.lat) + ', ' +
str(loc.lng) + '\n')
# gotta feed the route to routemanager... TODO: without recalc...
proc = thread_pool.apply_async(route_manager.recalc_route, args=(1, 99999999,
0, False))
areas_procs[area["name"]] = proc
# log.error("Calculated route, appending another coord and recalculating")
area_dict["routemanager"] = route_manager
areas[area["name"]] = area_dict
for area in areas_procs.keys():
to_be_checked = areas_procs[area]
log.debug(to_be_checked)
to_be_checked.get()
thread_pool.close()
thread_pool.join()
return areas
class RouteManagerBase(ABC):
def __init__(self,
db_wrapper,
coords,
max_radius,
max_coords_within_radius,
path_to_include_geofence,
path_to_exclude_geofence,
routefile,
mode=None,
init=False,
name="unknown",
settings=None):
self.db_wrapper = db_wrapper
self.init = init
self.name = name
self._coords_unstructured = coords
self.geofence_helper = GeofenceHelper(path_to_include_geofence,
path_to_exclude_geofence)
self._routefile = routefile
self._max_radius = max_radius
self._max_coords_within_radius = max_coords_within_radius
self.settings = settings
self.mode = mode
self._is_started = False
# we want to store the workers using the routemanager
self._workers_registered = []
self._workers_registered_mutex = Lock()
self._last_round_prio = False
self._manager_mutex = RLock()
self._round_started_time = None
if coords is not None:
if init:
fenced_coords = coords
else:
fenced_coords = self.geofence_helper.get_geofenced_coordinates(
coords)
self._route = getJsonRoute(fenced_coords, max_radius,
max_coords_within_radius, routefile)
else:
self._route = None
self._current_index_of_route = 0
self._init_mode_rounds = 0
if self.settings is not None:
self.delay_after_timestamp_prio = self.settings.get(
"delay_after_prio_event", None)
self.starve_route = self.settings.get("starve_route", False)
else:
self.delay_after_timestamp_prio = None
self.starve_route = False
# initialize priority queue variables
self._prio_queue = None
self._update_prio_queue_thread = None
self._stop_update_thread = Event()
def __del__(self):
if self._update_prio_queue_thread is not None:
self._stop_update_thread.set()
self._update_prio_queue_thread.join()
def clear_coords(self):
self._manager_mutex.acquire()
self._coords_unstructured = None
self._manager_mutex.release()
def register_worker(self, worker_name):
self._workers_registered_mutex.acquire()
try:
if worker_name in self._workers_registered:
log.info("Worker %s already registered to routemanager %s" %
(str(worker_name), str(self.name)))
return False
else:
log.info("Worker %s registering to routemanager %s" %
(str(worker_name), str(self.name)))
self._workers_registered.append(worker_name)
return True
finally:
self._workers_registered_mutex.release()
def unregister_worker(self, worker_name):
self._workers_registered_mutex.acquire()
try:
if worker_name in self._workers_registered:
log.info("Worker %s unregistering from routemanager %s" %
(str(worker_name), str(self.name)))
self._workers_registered.remove(worker_name)
else:
# TODO: handle differently?
log.info(
"Worker %s failed unregistering from routemanager %s since subscription was previously "
"lifted" % (str(worker_name), str(self.name)))
if len(self._workers_registered) == 0 and self._is_started:
log.info(
"Routemanager %s does not have any subscribing workers anymore, calling stop"
% str(self.name))
self._quit_route()
finally:
self._workers_registered_mutex.release()
def _check_started(self):
return self._is_started
def _start_priority_queue(self):
if (self._update_prio_queue_thread is None
and (self.delay_after_timestamp_prio is not None
or self.mode == "iv_mitm")
and not self.mode == "pokestops"):
self._prio_queue = []
if self.mode not in ["iv_mitm", "pokestops"]:
self.clustering_helper = ClusteringHelper(
self._max_radius, self._max_coords_within_radius,
self._cluster_priority_queue_criteria())
self._update_prio_queue_thread = Thread(
name="prio_queue_update_" + self.name,
target=self._update_priority_queue_loop)
self._update_prio_queue_thread.daemon = False
self._update_prio_queue_thread.start()
# list_coords is a numpy array of arrays!
def add_coords_numpy(self, list_coords):
fenced_coords = self.geofence_helper.get_geofenced_coordinates(
list_coords)
self._manager_mutex.acquire()
if self._coords_unstructured is None:
self._coords_unstructured = fenced_coords
else:
self._coords_unstructured = np.concatenate(
(self._coords_unstructured, fenced_coords))
self._manager_mutex.release()
def add_coords_list(self, list_coords):
to_be_appended = np.zeros(shape=(len(list_coords), 2))
for i in range(len(list_coords)):
to_be_appended[i][0] = list_coords[i][0]
to_be_appended[i][1] = list_coords[i][1]
self.add_coords_numpy(to_be_appended)
@staticmethod
def calculate_new_route(coords,
max_radius,
max_coords_within_radius,
routefile,
delete_old_route,
num_procs=0):
if delete_old_route and os.path.exists(routefile + ".calc"):
log.debug("Deleting routefile...")
os.remove(routefile + ".calc")
new_route = getJsonRoute(coords,
max_radius,
max_coords_within_radius,
num_processes=num_procs,
routefile=routefile)
return new_route
def recalc_route(self,
max_radius,
max_coords_within_radius,
num_procs=1,
delete_old_route=False,
nofile=False):
current_coords = self._coords_unstructured
if nofile:
routefile = None
else:
routefile = self._routefile
new_route = RouteManagerBase.calculate_new_route(
current_coords, max_radius, max_coords_within_radius, routefile,
delete_old_route, num_procs)
self._manager_mutex.acquire()
self._route = new_route
self._current_index_of_route = 0
self._manager_mutex.release()
def _update_priority_queue_loop(self):
if self._priority_queue_update_interval(
) is None or self._priority_queue_update_interval() == 0:
return
while not self._stop_update_thread.is_set():
# retrieve the latest hatches from DB
# newQueue = self._db_wrapper.get_next_raid_hatches(self._delayAfterHatch, self._geofenceHelper)
new_queue = self._retrieve_latest_priority_queue()
self._merge_priority_queue(new_queue)
time.sleep(self._priority_queue_update_interval())
def _merge_priority_queue(self, new_queue):
if new_queue is not None:
self._manager_mutex.acquire()
merged = set(new_queue + self._prio_queue)
merged = list(merged)
merged = self._filter_priority_queue_internal(merged)
heapq.heapify(merged)
self._prio_queue = merged
self._manager_mutex.release()
log.info("New priorityqueue: %s" % merged)
def date_diff_in_seconds(self, dt2, dt1):
timedelta = dt2 - dt1
return timedelta.days * 24 * 3600 + timedelta.seconds
def dhms_from_seconds(self, seconds):
minutes, seconds = divmod(seconds, 60)
hours, minutes = divmod(minutes, 60)
# days, hours = divmod(hours, 24)
return hours, minutes, seconds
def _get_round_finished_string(self):
round_finish_time = datetime.now()
round_completed_in = (
"%d hours, %d minutes, %d seconds" % (self.dhms_from_seconds(
self.date_diff_in_seconds(round_finish_time,
self._round_started_time))))
return round_completed_in
@abstractmethod
def _retrieve_latest_priority_queue(self):
"""
Method that's supposed to return a plain list containing (timestamp, Location) of the next events of interest
:return:
"""
pass
@abstractmethod
def _start_routemanager(self):
"""
Starts priority queue or whatever the implementations require
:return:
"""
pass
@abstractmethod
def _quit_route(self):
"""
Killing the Route Thread
:return:
"""
pass
@abstractmethod
def _get_coords_post_init(self):
"""
Return list of coords to be fetched and used for routecalc
:return:
"""
pass
@abstractmethod
def _check_coords_before_returning(self, lat, lng):
"""
Return list of coords to be fetched and used for routecalc
:return:
"""
pass
@abstractmethod
def _recalc_route_workertype(self):
"""
Return a new route for worker
:return:
"""
pass
@abstractmethod
def _get_coords_after_finish_route(self):
"""
Return list of coords to be fetched after finish a route
:return:
"""
pass
@abstractmethod
def _cluster_priority_queue_criteria(self):
"""
If you do not want to have any filtering, simply return 0, 0, otherwise simply
return timedelta_seconds, distance
:return:
"""
@abstractmethod
def _priority_queue_update_interval(self):
"""
The time to sleep in between consecutive updates of the priority queue
:return:
"""
def _filter_priority_queue_internal(self, latest):
"""
Filter through the internal priority queue and cluster events within the timedelta and distance returned by
_cluster_priority_queue_criteria
:return:
"""
# timedelta_seconds = self._cluster_priority_queue_criteria()
if self.mode == "iv_mitm":
# exclude IV prioQ to also pass encounterIDs since we do not pass additional information through when
# clustering
return latest
delete_seconds_passed = 0
if self.settings is not None:
delete_seconds_passed = self.settings.get(
"remove_from_queue_backlog", 0)
if delete_seconds_passed is not None:
delete_before = time.time() - delete_seconds_passed
else:
delete_before = 0
latest = [
to_keep for to_keep in latest if not to_keep[0] < delete_before
]
# TODO: sort latest by modified flag of event
# merged = self._merge_queue(latest, self._max_radius, 2, timedelta_seconds)
merged = self.clustering_helper.get_clustered(latest)
return merged
def get_next_location(self):
log.debug("get_next_location of %s called" % str(self.name))
if not self._is_started:
log.info("Starting routemanager %s in get_next_location" %
str(self.name))
self._start_routemanager()
next_lat, next_lng = 0, 0
# first check if a location is available, if not, block until we have one...
got_location = False
while not got_location:
log.debug("%s: Checking if a location is available..." %
str(self.name))
self._manager_mutex.acquire()
got_location = (self._prio_queue is not None
and len(self._prio_queue) > 0 or
(self._route is not None and len(self._route) > 0))
self._manager_mutex.release()
if not got_location:
log.debug("%s: No location available yet" % str(self.name))
time.sleep(0.5)
log.debug(
"%s: Location available, acquiring lock and trying to return location"
% str(self.name))
self._manager_mutex.acquire()
# check priority queue for items of priority that are past our time...
# if that is not the case, simply increase the index in route and return the location on route
# determine whether we move to the next location or the prio queue top's item
if (self.delay_after_timestamp_prio is not None
and ((not self._last_round_prio or self.starve_route)
and self._prio_queue and len(self._prio_queue) > 0
and self._prio_queue[0][0] < time.time())):
log.debug("%s: Priority event" % str(self.name))
next_stop = heapq.heappop(self._prio_queue)[1]
next_lat = next_stop.lat
next_lng = next_stop.lng
self._last_round_prio = True
log.info(
"Round of route %s is moving to %s, %s for a priority event" %
(str(self.name), str(next_lat), str(next_lng)))
else:
log.debug("%s: Moving on with route" % str(self.name))
if self._current_index_of_route == 0:
if self._round_started_time is not None:
log.info(
"Round of route %s reached the first spot again. It took %s"
% (str(self.name),
str(self._get_round_finished_string())))
self._round_started_time = datetime.now()
if len(self._route) == 0: return None
log.info("Round of route %s started at %s" %
(str(self.name), str(self._round_started_time)))
# continue as usual
if self._current_index_of_route < len(self._route):
log.info(
"%s: Moving on with location %s [%s/%s]" %
(str(self.name), self._route[self._current_index_of_route],
self._current_index_of_route + 1, len(self._route)))
next_lat = self._route[self._current_index_of_route]['lat']
next_lng = self._route[self._current_index_of_route]['lng']
self._current_index_of_route += 1
if self.init and self._current_index_of_route >= len(self._route):
self._init_mode_rounds += 1
if self.init and self._current_index_of_route >= len(self._route) and \
self._init_mode_rounds >= int(self.settings.get("init_mode_rounds", 1)):
# we are done with init, let's calculate a new route
log.warning(
"Init of %s done, it took %s, calculating new route..." %
(str(self.name), self._get_round_finished_string()))
self.clear_coords()
coords = self._get_coords_post_init()
log.debug("Setting %s coords to as new points in route of %s" %
(str(len(coords)), str(self.name)))
self.add_coords_list(coords)
log.debug("Route of %s is being calculated" % str(self.name))
self._recalc_route_workertype()
self.init = False
self.change_init_mapping(self.name)
self._manager_mutex.release()
return self.get_next_location()
elif self._current_index_of_route == len(self._route):
log.info('Reaching last coord of route')
elif self._current_index_of_route > len(self._route):
self._current_index_of_route = 0
coords_after_round = self._get_coords_after_finish_route()
# TODO: check isinstance list?
if coords_after_round is not None:
self.clear_coords()
coords = coords_after_round
self.add_coords_list(coords)
self._recalc_route_workertype()
if len(self._route) == 0: return None
next_lat = self._route[self._current_index_of_route]['lat']
next_lng = self._route[self._current_index_of_route]['lng']
self._manager_mutex.release()
return Location(next_lat, next_lng)
self._manager_mutex.release()
return self.get_next_location()
self._last_round_prio = False
log.info(
"%s done grabbing next coord, releasing lock and returning location: %s, %s"
% (str(self.name), str(next_lat), str(next_lng)))
self._manager_mutex.release()
if self._check_coords_before_returning(next_lat, next_lng):
return Location(next_lat, next_lng)
else:
return self.get_next_location()
def del_from_route(self):
log.debug(
"%s: Location available, acquiring lock and trying to return location"
% str(self.name))
self._manager_mutex.acquire()
log.info('Removing coords from Route')
self._route.pop(int(self._current_index_of_route) - 1)
self._current_index_of_route -= 1
if len(self._route) == 0:
log.info('No more coords are available... Sleeping.')
self._manager_mutex.release()
def change_init_mapping(self, name_area):
with open('configs/mappings.json') as f:
vars = json.load(f)
for var in vars['areas']:
if (var['name']) == name_area:
var['init'] = bool(False)
with open('mappings.json', 'w') as outfile:
json.dump(vars, outfile, indent=4, sort_keys=True)
def get_route_status(self):
if self._route:
return self._current_index_of_route, len(self._route)
return self._current_index_of_route, self._current_index_of_route
def __init__(self, db_wrapper: DbWrapperBase, dbm: DataManager, area_id: str, coords: List[Location], max_radius: float,
max_coords_within_radius: int, path_to_include_geofence: str, path_to_exclude_geofence: str,
routefile: str, mode=None, init: bool = False, name: str = "unknown", settings: dict = None,
level: bool = False, calctype: str = "optimized", useS2: bool = False, S2level: int = 15, joinqueue = None):
self.db_wrapper: DbWrapperBase = db_wrapper
self.init: bool = init
self.name: str = name
self._data_manager = dbm
self.useS2: bool = useS2
self.S2level: int = S2level
self.area_id = area_id
self._coords_unstructured: List[Location] = coords
self.geofence_helper: GeofenceHelper = GeofenceHelper(
path_to_include_geofence, path_to_exclude_geofence)
self._routefile = os.path.join(args.file_path, routefile)
self._max_radius: float = max_radius
self._max_coords_within_radius: int = max_coords_within_radius
self.settings: dict = settings
self.mode = mode
self._is_started: bool = False
self._first_started = False
self._current_route_round_coords: List[Location] = []
self._start_calc: bool = False
self._positiontyp = {}
self._coords_to_be_ignored = set()
self._level = level
self._calctype = calctype
self._overwrite_calculation: bool = False
self._stops_not_processed: Dict[Location, int] = {}
self._routepool: Dict[str, RoutePoolEntry] = {}
self._roundcount: int = 0
self._joinqueue = joinqueue
# we want to store the workers using the routemanager
self._workers_registered: List[str] = []
self._workers_registered_mutex = RLock()
# waiting till routepool is filled up
self._workers_fillup_mutex = RLock()
self._last_round_prio = {}
self._manager_mutex = RLock()
self._round_started_time = None
self._route: List[Location] = []
if coords is not None:
if init:
fenced_coords = coords
else:
fenced_coords = self.geofence_helper.get_geofenced_coordinates(
coords)
new_coords = getJsonRoute(
fenced_coords, max_radius, max_coords_within_radius, routefile,
algorithm=calctype)
for coord in new_coords:
self._route.append(Location(coord["lat"], coord["lng"]))
self._current_index_of_route = 0
self._init_mode_rounds = 0
if self.settings is not None:
self.delay_after_timestamp_prio = self.settings.get(
"delay_after_prio_event", None)
self.starve_route = self.settings.get("starve_route", False)
else:
self.delay_after_timestamp_prio = None
self.starve_route = False
# initialize priority queue variables
self._prio_queue = None
self._update_prio_queue_thread = None
self._check_routepools_thread = None
self._stop_update_thread = Event()
def main():
args = Args()
initLogging(args)
if len(sys.argv) != 2:
logger.error("usage: remove_all_spawns_within_geofence.py GEOFENCE_FILENAME")
sys.exit(1)
LocationWithID = collections.namedtuple('Location', ['lat', 'lng', 'spawnpoint'])
geofence_filename = sys.argv[1]
#print("Argument: '%s'" % (geofence_filename))
# no .txt, add it
if ".txt" not in geofence_filename:
geofence_filename = geofence_filename+".txt"
# no / in filename, probably not an absolute path, append standard MAD path
if "/" not in geofence_filename:
geofence_filename = "../configs/geofences/" + geofence_filename
logger.info("Trying to use file: {}", geofence_filename)
if not os.path.isfile(geofence_filename):
logger.error("Geofence file {} not found, exit", geofence_filename)
sys.exit(1)
geofence_helper = GeofenceHelper(geofence_filename, None)
minLat, minLon, maxLat, maxLon = geofence_helper.get_polygon_from_fence()
query = (
"SELECT latitude, longitude, spawnpoint "
"FROM trs_spawn "
"WHERE (latitude >= {} AND longitude >= {} "
"AND latitude <= {} AND longitude <= {}) "
).format(minLat, minLon, maxLat, maxLon)
delete_query = (
"DELETE FROM trs_spawn "
"WHERE spawnpoint = {} "
)
list_of_coords: List[LocationWithID] = []
dbip = get_value_for(r'\s+dbip:\s+([^\s]+)')
dbport = get_value_for(r'\s+dbport:\s+([^.\s]*)', False)
if dbport is None: # if dbport is not set, use default
dbport = '3306'
dbusername = get_value_for(r'\s+dbusername:\s+([^.\s]*)')
dbpassword = get_value_for(r'\s+dbpassword:\s+([^.\s]*)')
dbname = get_value_for(r'\s+dbname:\s+([^.\s]*)')
#print("Successfully parsed config.ini, using values:")
#print("dbport: %s" % dbport)
#print("dbusername: %s" % dbusername)
#print("dbname: %s" % dbname)
#print("dbip: %s" % dbip)
connection = mysql.connector.connect(
host = dbip,
port = dbport,
user = dbusername,
passwd = dbpassword,
database = dbname)
cursor = connection.cursor()
cursor.execute(query)
res = cursor.fetchall()
for (latitude, longitude, spawnpoint) in res:
list_of_coords.append(LocationWithID(latitude, longitude, spawnpoint))
geofenced_coords = geofence_helper.get_geofenced_coordinates(list_of_coords)
spawnpointcount = len(geofenced_coords)
for coords in geofenced_coords:
sql = delete_query.format(coords.spawnpoint)
cursor.execute(sql)
#print(sql)
connection.commit()
cursor.close()
connection.close()
logger.success("Done, deleted {} spawnpoints", spawnpointcount)
class RouteManagerBase(ABC):
def __init__(self, db_wrapper, coords, max_radius, max_coords_within_radius, path_to_include_geofence,
path_to_exclude_geofence, routefile, init=False,
name="unknown", settings=None):
self.db_wrapper = db_wrapper
self.init = init
self.name = name
self._coords_unstructured = coords
self.geofence_helper = GeofenceHelper(path_to_include_geofence, path_to_exclude_geofence)
self._routefile = routefile
self._max_radius = max_radius
self._max_coords_within_radius = max_coords_within_radius
self.settings = settings
self._last_round_prio = False
self._manager_mutex = Lock()
self._round_started_time = None
if coords is not None:
if init:
fenced_coords = coords
else:
fenced_coords = self.geofence_helper.get_geofenced_coordinates(coords)
self._route = getJsonRoute(fenced_coords, max_radius, max_coords_within_radius, routefile)
else:
self._route = None
self._current_index_of_route = 0
if settings is not None:
self.delay_after_timestamp_prio = settings.get("delay_after_prio_event", None)
self.starve_route = settings.get("starve_route", False)
else:
self.delay_after_timestamp_prio = None
self.starve_route = False
if self.delay_after_timestamp_prio is not None:
self._prio_queue = []
self.clustering_helper = ClusteringHelper(self._max_radius,
self._max_coords_within_radius,
self._cluster_priority_queue_criteria())
self._stop_update_thread = Event()
self._update_prio_queue_thread = Thread(name="prio_queue_update_" + name,
target=self._update_priority_queue_loop)
self._update_prio_queue_thread.daemon = True
self._update_prio_queue_thread.start()
else:
self._prio_queue = None
def __del__(self):
if self.delay_after_timestamp_prio:
self._stop_update_thread.set()
self._update_prio_queue_thread.join()
def clear_coords(self):
self._manager_mutex.acquire()
self._coords_unstructured = None
self._manager_mutex.release()
# list_coords is a numpy array of arrays!
def add_coords_numpy(self, list_coords):
fenced_coords = self.geofence_helper.get_geofenced_coordinates(list_coords)
self._manager_mutex.acquire()
if self._coords_unstructured is None:
self._coords_unstructured = fenced_coords
else:
self._coords_unstructured = np.concatenate((self._coords_unstructured, fenced_coords))
self._manager_mutex.release()
def add_coords_list(self, list_coords):
to_be_appended = np.zeros(shape=(len(list_coords), 2))
for i in range(len(list_coords)):
to_be_appended[i][0] = list_coords[i][0]
to_be_appended[i][1] = list_coords[i][1]
self.add_coords_numpy(to_be_appended)
@staticmethod
def calculate_new_route(coords, max_radius, max_coords_within_radius, routefile, delete_old_route, num_procs=0):
if delete_old_route and os.path.exists(routefile + ".calc"):
log.debug("Deleting routefile...")
os.remove(routefile + ".calc")
new_route = getJsonRoute(coords, max_radius, max_coords_within_radius, num_processes=num_procs,
routefile=routefile)
return new_route
def recalc_route(self, max_radius, max_coords_within_radius, num_procs=1, delete_old_route=False):
current_coords = self._coords_unstructured
routefile = self._routefile
new_route = RouteManagerBase.calculate_new_route(current_coords, max_radius, max_coords_within_radius,
routefile, delete_old_route, num_procs)
self._manager_mutex.acquire()
self._route = new_route
self._current_index_of_route = 0
self._manager_mutex.release()
def _update_priority_queue_loop(self):
while not self._stop_update_thread.is_set():
# retrieve the latest hatches from DB
# newQueue = self._db_wrapper.get_next_raid_hatches(self._delayAfterHatch, self._geofenceHelper)
new_queue = self._retrieve_latest_priority_queue()
self._merge_priority_queue(new_queue)
time.sleep(300)
def _merge_priority_queue(self, new_queue):
self._manager_mutex.acquire()
merged = list(set(new_queue + self._prio_queue))
merged = self._filter_priority_queue_internal(merged)
heapq.heapify(merged)
self._prio_queue = merged
self._manager_mutex.release()
log.info("New priorityqueue: %s" % merged)
def date_diff_in_seconds(self, dt2, dt1):
timedelta = dt2 - dt1
return timedelta.days * 24 * 3600 + timedelta.seconds
def dhms_from_seconds(self, seconds):
minutes, seconds = divmod(seconds, 60)
hours, minutes = divmod(minutes, 60)
# days, hours = divmod(hours, 24)
return hours, minutes, seconds
def _get_round_finished_string(self):
round_finish_time = datetime.now()
round_completed_in = (
"%d hours, %d minutes, %d seconds" % (
self.dhms_from_seconds(
self.date_diff_in_seconds(round_finish_time, self._round_started_time)
)
)
)
return round_completed_in
@abstractmethod
def _retrieve_latest_priority_queue(self):
"""
Method that's supposed to return a plain list containing (timestamp, Location) of the next events of interest
:return:
"""
pass
@abstractmethod
def _get_coords_post_init(self):
"""
Return list of coords to be fetched and used for routecalc
:return:
"""
pass
@abstractmethod
def _cluster_priority_queue_criteria(self):
"""
If you do not want to have any filtering, simply return 0, 0, otherwise simply
return timedelta_seconds, distance
:return:
"""
def _filter_priority_queue_internal(self, latest):
"""
Filter through the internal priority queue and cluster events within the timedelta and distance returned by
_cluster_priority_queue_criteria
:return:
"""
# timedelta_seconds = self._cluster_priority_queue_criteria()
delete_seconds_passed = 0
if self.settings is not None:
delete_seconds_passed = self.settings.get("remove_from_queue_backlog", 0)
if delete_seconds_passed is not None:
delete_before = time.time() - delete_seconds_passed
else:
delete_before = 0
latest = [to_keep for to_keep in latest if not to_keep[0] < delete_before]
# TODO: sort latest by modified flag of event
# merged = self._merge_queue(latest, self._max_radius, 2, timedelta_seconds)
merged = self.clustering_helper.get_clustered(latest)
return merged
def get_next_location(self):
log.debug("get_next_location of %s called" % str(self.name))
next_lat, next_lng = 0, 0
# first check if a location is available, if not, block until we have one...
got_location = False
while not got_location:
log.debug("%s: Checking if a location is available..." % str(self.name))
self._manager_mutex.acquire()
got_location = self._prio_queue is not None and len(self._prio_queue) > 0 or len(self._route) > 0
self._manager_mutex.release()
if not got_location:
log.debug("%s: No location available yet" % str(self.name))
time.sleep(0.5)
log.debug("%s: Location available, acquiring lock and trying to return location")
self._manager_mutex.acquire()
# check priority queue for items of priority that are past our time...
# if that is not the case, simply increase the index in route and return the location on route
# determine whether we move to the next location or the prio queue top's item
if (self.delay_after_timestamp_prio is not None and ((not self._last_round_prio or self.starve_route)
and len(self._prio_queue) > 0
and self._prio_queue[0][0] < time.time())):
log.debug("%s: Priority event" % str(self.name))
next_stop = heapq.heappop(self._prio_queue)[1]
next_lat = next_stop.lat
next_lng = next_stop.lng
self._last_round_prio = True
log.info("Round of route %s is moving to %s, %s for a priority event"
% (str(self.name), str(next_lat), str(next_lng)))
else:
log.debug("%s: Moving on with route" % str(self.name))
if self._current_index_of_route == 0:
if self._round_started_time is not None:
log.info("Round of route %s reached the first spot again. It took %s"
% (str(self.name), str(self._get_round_finished_string())))
self._round_started_time = datetime.now()
log.info("Round of route %s started at %s" % (str(self.name), str(self._round_started_time)))
# continue as usual
log.info("Moving on with location %s" % self._route[self._current_index_of_route])
next_lat = self._route[self._current_index_of_route]['lat']
next_lng = self._route[self._current_index_of_route]['lng']
self._current_index_of_route += 1
if self.init and self._current_index_of_route >= len(self._route):
# we are done with init, let's calculate a new route
log.warning("Init of %s done, it took %s, calculating new route..."
% (str(self.name), self._get_round_finished_string()))
self._manager_mutex.release()
self.clear_coords()
coords = self._get_coords_post_init()
log.debug("Setting %s coords to as new points in route of %s"
% (str(len(coords)), str(self.name)))
self.add_coords_list(coords)
log.debug("Route of %s is being calculated" % str(self.name))
self.recalc_route(self._max_radius, self._max_coords_within_radius, 1, True)
self.init = False
return self.get_next_location()
elif self._current_index_of_route >= len(self._route):
self._current_index_of_route = 0
self._last_round_prio = False
log.info("%s done grabbing next coord, releasing lock and returning location: %s, %s"
% (str(self.name), str(next_lat), str(next_lng)))
self._manager_mutex.release()
return Location(next_lat, next_lng)
def __get_latest_routemanagers(self) -> Optional[Dict[str, dict]]:
global mode_mapping
areas: Optional[Dict[str, dict]] = {}
if self.__configmode:
return areas
raw_areas = self.__data_manager.get_root_resource('area')
thread_pool = ThreadPool(processes=4)
areas_procs = {}
for area_id, area_true in raw_areas.items():
area = area_true.get_resource()
if area["geofence_included"] is None:
raise RuntimeError("Cannot work without geofence_included")
try:
geofence_included = self.__data_manager.get_resource(
'geofence', identifier=area["geofence_included"])
except:
raise RuntimeError(
"geofence_included for area '{}' is specified but does not exist ('{}')."
.format(area["name"], geofence_included))
geofence_excluded_raw_path = area.get("geofence_excluded", None)
try:
if geofence_excluded_raw_path is not None:
geofence_excluded = self.__data_manager.get_resource(
'geofence', identifier=geofence_excluded_raw_path)
else:
geofence_excluded = None
except:
raise RuntimeError(
"geofence_excluded for area '{}' is specified but file does not exist ('{}')."
.format(area["name"], geofence_excluded_raw_path))
area_dict = {
"mode": area_true.area_type,
"geofence_included": geofence_included,
"geofence_excluded": geofence_excluded,
"routecalc": area["routecalc"],
"name": area['name']
}
# also build a routemanager for each area...
# grab coords
# first check if init is false, if so, grab the coords from DB
# coords = np.loadtxt(area["coords"], delimiter=',')
geofence_helper = GeofenceHelper(geofence_included,
geofence_excluded)
mode = area_true.area_type
# build routemanagers
# map iv list to ids
if area.get('settings',
None) is not None and 'mon_ids_iv' in area['settings']:
# replace list name
area['settings']['mon_ids_iv_raw'] = \
self.get_monlist(area['settings'].get('mon_ids_iv', None), area.get("name", "unknown"))
route_resource = self.__data_manager.get_resource(
'routecalc', identifier=area["routecalc"])
route_manager = RouteManagerFactory.get_routemanager(
self.__db_wrapper,
self.__data_manager,
area_id,
None,
mode_mapping.get(mode, {}).get("range", 0),
mode_mapping.get(mode, {}).get("max_count", 99999999),
geofence_included,
path_to_exclude_geofence=geofence_excluded,
mode=mode,
settings=area.get("settings", None),
init=area.get("init", False),
name=area.get("name", "unknown"),
level=area.get("level", False),
coords_spawns_known=area.get("coords_spawns_known", False),
routefile=route_resource,
calctype=area.get("route_calc_algorithm", "optimized"),
joinqueue=self.join_routes_queue,
S2level=mode_mapping.get(mode, {}).get("s2_cell_level", 30))
if mode not in ("iv_mitm", "idle"):
coords = self.__fetch_coords(
mode,
geofence_helper,
coords_spawns_known=area.get("coords_spawns_known", False),
init=area.get("init", False),
range_init=mode_mapping.get(area_true.area_type,
{}).get("range_init", 630),
including_stops=area.get("including_stops", False))
route_manager.add_coords_list(coords)
max_radius = mode_mapping[area_true.area_type]["range"]
max_count_in_radius = mode_mapping[
area_true.area_type]["max_count"]
if not area.get("init", False):
logger.info("Initializing area {}", area["name"])
proc = thread_pool.apply_async(
route_manager.initial_calculation,
args=(max_radius, max_count_in_radius, 0, False))
areas_procs[area_id] = proc
else:
logger.info("Init mode enabled. Going row-based for {}",
str(area.get("name", "unknown")))
# we are in init, let's write the init route to file to make it visible in madmin
calc_coords = []
if area["routecalc"] is not None:
for loc in coords:
calc_coord = '%s,%s' % (str(loc.lat), str(loc.lng))
calc_coords.append(calc_coord)
route_resource['routefile'] = calc_coords
route_resource.save()
# gotta feed the route to routemanager... TODO: without recalc...
proc = thread_pool.apply_async(route_manager.recalc_route,
args=(1, 99999999, 0,
False))
areas_procs[area_id] = proc
area_dict["routemanager"] = route_manager
areas[area_id] = area_dict
for area in areas_procs.keys():
to_be_checked = areas_procs[area]
to_be_checked.get()
thread_pool.close()
thread_pool.join()
return areas
def __init__(self,
db_wrapper: DbWrapperBase,
coords: List[Location],
max_radius: float,
max_coords_within_radius: int,
path_to_include_geofence: str,
path_to_exclude_geofence: str,
routefile: str,
mode=None,
init: bool = False,
name: str = "unknown",
settings: dict = None):
self.db_wrapper: DbWrapperBase = db_wrapper
self.init: bool = init
self.name: str = name
self._coords_unstructured: List[Location] = coords
self.geofence_helper: GeofenceHelper = GeofenceHelper(
path_to_include_geofence, path_to_exclude_geofence)
self._routefile = os.path.join(args.file_path, routefile)
self._max_radius: float = max_radius
self._max_coords_within_radius: int = max_coords_within_radius
self.settings: dict = settings
self.mode = mode
self._is_started: bool = False
self._first_started = False
self._route_queue = Queue()
self._start_calc: bool = False
self._rounds = {}
self._positiontyp = {}
self._coords_to_be_ignored = set()
# we want to store the workers using the routemanager
self._workers_registered: List[WorkerBase] = []
self._workers_registered_mutex = Lock()
self._last_round_prio = {}
self._manager_mutex = RLock()
self._round_started_time = None
self._route: List[Location] = []
if coords is not None:
if init:
fenced_coords = coords
else:
fenced_coords = self.geofence_helper.get_geofenced_coordinates(
coords)
new_coords = getJsonRoute(fenced_coords, max_radius,
max_coords_within_radius, routefile)
for coord in new_coords:
self._route.append(Location(coord["lat"], coord["lng"]))
self._current_index_of_route = 0
self._init_mode_rounds = 0
if self.settings is not None:
self.delay_after_timestamp_prio = self.settings.get(
"delay_after_prio_event", None)
self.starve_route = self.settings.get("starve_route", False)
else:
self.delay_after_timestamp_prio = None
self.starve_route = False
# initialize priority queue variables
self._prio_queue = None
self._update_prio_queue_thread = None
self._stop_update_thread = Event()
class RouteManager:
def __init__(self,
db_wrapper,
coords,
maxRadius,
maxCoordsWithinRadius,
pathToIncludeGeofence,
pathToExcludeGeofence,
routefile,
coords_spawns_known=False,
delayAfterHatch=None,
init=False,
mode=None,
settings=None,
name="unknown"):
# TODO: handle mode=None...
# first filter the coords with the geofence given
self.init = init
self.mode = mode
self.name = name
self.settings = settings
self.coords_spawns_known = coords_spawns_known
self.__coords_unstructured = coords
self.__geofenceHelper = GeofenceHelper(pathToIncludeGeofence,
pathToExcludeGeofence)
self.__db_wrapper = db_wrapper
self.__managerMutex = Lock()
self.__lastRoundEggHatch = False # boolean to prevent starvation in a simple way...
self.__routefile = routefile
self.__max_radius = maxRadius
self.__max_coords_within_radius = maxCoordsWithinRadius
self.__round_started_time = None
if coords is not None:
if init:
fencedCoords = coords
else:
fencedCoords = self.__geofenceHelper.get_geofenced_coordinates(
coords)
self.__route = getJsonRoute(fencedCoords, maxRadius,
maxCoordsWithinRadius, routefile)
else:
self.__route = None
self.__currentIndexOfRoute = 0
# heapq of hatched eggs
self.__delayAfterHatch = delayAfterHatch
if self.__delayAfterHatch is not None:
self.__raidQueue = []
self.__stopUpdateThread = Event()
self.__updateRaidQueueThread = Thread(
name='raidQUpdate', target=self.__updatePriorityQueueLoop)
self.__updateRaidQueueThread.daemon = False
self.__updateRaidQueueThread.start()
else:
self.__raidQueue = None
def __del__(self):
if self.__delayAfterHatch:
self.__stopUpdateThread.set()
self.__updateRaidQueueThread.join()
def clear_coords(self):
self.__managerMutex.acquire()
self.__coords_unstructured = None
self.__managerMutex.release()
# list_coords is a numpy array of arrays!
def add_coords_numpy(self, list_coords):
fenced_coords = self.__geofenceHelper.get_geofenced_coordinates(
list_coords)
self.__managerMutex.acquire()
if self.__coords_unstructured is None:
self.__coords_unstructured = fenced_coords
else:
self.__coords_unstructured = np.concatenate(
(self.__coords_unstructured, fenced_coords))
self.__managerMutex.release()
def add_coords_list(self, list_coords):
to_be_appended = np.zeros(shape=(len(list_coords), 2))
for i in range(len(list_coords)):
to_be_appended[i][0] = list_coords[i][0]
to_be_appended[i][1] = list_coords[i][1]
self.add_coords_numpy(to_be_appended)
@staticmethod
def calculate_new_route(coords,
max_radius,
max_coords_within_radius,
routefile,
delete_old_route,
num_procs=0):
if delete_old_route and os.path.exists(routefile + ".calc"):
log.debug("Deleting routefile...")
os.remove(routefile + ".calc")
new_route = getJsonRoute(coords,
max_radius,
max_coords_within_radius,
num_processes=num_procs,
routefile=routefile)
return new_route
def recalc_route(self,
max_radius,
max_coords_within_radius,
num_procs=1,
delete_old_route=False):
current_coords = self.__coords_unstructured
routefile = self.__routefile
new_route = RouteManager.calculate_new_route(
current_coords, max_radius, max_coords_within_radius, routefile,
delete_old_route, num_procs)
self.__managerMutex.acquire()
self.__route = new_route
self.__currentIndexOfRoute = 0
self.__managerMutex.release()
def __updatePriorityQueueLoop(self):
while not self.__stopUpdateThread.is_set():
# retrieve the latest hatches from DB
newQueue = self.__db_wrapper.get_next_raid_hatches(
self.__delayAfterHatch, self.__geofenceHelper)
self.__mergeRaidQueue(newQueue)
time.sleep(300)
def __mergeRaidQueue(self, newQueue):
self.__managerMutex.acquire()
merged = list(set(newQueue + self.__raidQueue))
heapq.heapify(merged)
self.__raidQueue = merged
self.__managerMutex.release()
log.info("New raidqueue: %s" % merged)
def getNextLocation(self):
# nextLocation = Location(None, None)
nextLat = 0
nextLng = 0
# first check if a location is available, if not, block until we have one... TODO
got_location = False
while not got_location:
self.__managerMutex.acquire()
got_location = self.__raidQueue is not None and len(
self.__raidQueue) > 0 or len(self.__route) > 0
self.__managerMutex.release()
if not got_location:
time.sleep(0.5)
self.__managerMutex.acquire()
# check raid queue for hatches, if none have passed, simply increase index and return the location in the route
# at index
# determine whether we move to the next gym or to the top of our priority queue
if self.__delayAfterHatch is not None and (
not self.__lastRoundEggHatch and len(self.__raidQueue) > 0
and self.__raidQueue[0][0] < time.time()):
nextStop = heapq.heappop(
self.__raidQueue)[1] # gets the location tuple
nextLat = nextStop.latitude
nextLng = nextStop.longitude
self.__lastRoundEggHatch = True
else:
if self.__currentIndexOfRoute == 0:
if self.__round_started_time is not None:
log.info(
"Round of route %s reached the first spot again. It took: %s"
% (str(self.name), self.get_round_finished_string()))
self.__round_started_time = datetime.now()
log.info("Round of route %s started at %s" %
(str(self.name), str(self.__round_started_time)))
# continue as usual
log.info('main: Moving on with gym at %s' %
self.__route[self.__currentIndexOfRoute])
nextLat = self.__route[self.__currentIndexOfRoute]['lat']
nextLng = self.__route[self.__currentIndexOfRoute]['lng']
self.__currentIndexOfRoute += 1
if self.init and self.__currentIndexOfRoute >= len(self.__route):
# we are done with init, let's calculate a new route
log.warning(
"Init of %s done, it took %s, calculating new route..." %
(str(self.name), self.get_round_finished_string()))
self.__managerMutex.release()
self.clear_coords()
if self.mode == "raids_ocr" or self.mode == "raids_mitm":
coords = self.__db_wrapper.gyms_from_db(
self.__geofenceHelper)
elif self.mode == "mon_mitm":
if self.coords_spawns_known:
log.info("Reading known Spawnpoints from DB")
coords = self.__db_wrapper.get_detected_spawns(
self.__geofenceHelper)
else:
log.info("Reading unknown Spawnpoints from DB")
coords = self.__db_wrapper.get_undetected_spawns(
self.__geofenceHelper)
else:
log.fatal("Mode not implemented yet: %s" % str(self.mode))
exit(1)
log.debug("Adding %s coords to list" % str(len(coords)))
self.add_coords_list(coords)
log.debug("Runnig calculation")
self.recalc_route(self.__max_radius,
self.__max_coords_within_radius, 1, True)
self.init = False
log.debug("Calling ourself to get coord...")
return self.getNextLocation()
elif self.__currentIndexOfRoute >= len(self.__route):
self.__currentIndexOfRoute = 0
self.__lastRoundEggHatch = False
self.__managerMutex.release()
return Location(nextLat, nextLng)
def date_diff_in_seconds(self, dt2, dt1):
timedelta = dt2 - dt1
return timedelta.days * 24 * 3600 + timedelta.seconds
def dhms_from_seconds(self, seconds):
minutes, seconds = divmod(seconds, 60)
hours, minutes = divmod(minutes, 60)
# days, hours = divmod(hours, 24)
return hours, minutes, seconds
def get_round_finished_string(self):
round_finish_time = datetime.now()
round_completed_in = (
"%d hours, %d minutes, %d seconds" % (self.dhms_from_seconds(
self.date_diff_in_seconds(round_finish_time,
self.__round_started_time))))
return round_completed_in
def __get_latest_routemanagers(self) -> Optional[Dict[str, dict]]:
global mode_mapping
areas: Optional[Dict[str, dict]] = {}
if self.__configmode:
return areas
area_arr = self.__raw_json["areas"]
thread_pool = ThreadPool(processes=4)
areas_procs = {}
for area in area_arr:
if area["geofence_included"] is None:
raise RuntimeError("Cannot work without geofence_included")
geofence_included = Path(area["geofence_included"])
if not geofence_included.is_file():
raise RuntimeError(
"geofence_included for area '{}' is specified but file does not exist ('{}').".format(
area["name"], geofence_included.resolve()
)
)
geofence_excluded_raw_path = area.get("geofence_excluded", None)
if geofence_excluded_raw_path is not None:
geofence_excluded = Path(geofence_excluded_raw_path)
if not geofence_excluded.is_file():
raise RuntimeError(
"geofence_excluded for area '{}' is specified but file does not exist ('{}').".format(
area["name"], geofence_excluded.resolve()
)
)
area_dict = {"mode": area["mode"],
"geofence_included": area["geofence_included"],
"geofence_excluded": area.get("geofence_excluded", None),
"routecalc": area["routecalc"]}
# also build a routemanager for each area...
# grab coords
# first check if init is false, if so, grab the coords from DB
# coords = np.loadtxt(area["coords"], delimiter=',')
geofence_helper = GeofenceHelper(
area["geofence_included"], area.get("geofence_excluded", None))
mode = area["mode"]
# build routemanagers
#map iv list to ids
if area.get('settings', None) is not None and 'mon_ids_iv' in area['settings']:
# replace list name
area['settings']['mon_ids_iv_raw'] = \
self.get_monlist(area['settings'].get('mon_ids_iv', None), area.get("name", "unknown"))
route_manager = RouteManagerFactory.get_routemanager(self.__db_wrapper, None,
mode_mapping.get(mode, {}).get("range", 0),
mode_mapping.get(mode, {}).get("max_count", 99999999),
area["geofence_included"],
area.get("geofence_excluded", None),
mode=mode, settings=area.get("settings", None),
init=area.get("init", False),
name=area.get("name", "unknown"),
level=area.get("level", False),
coords_spawns_known=area.get(
"coords_spawns_known", False),
routefile=area["routecalc"],
calctype=area.get("route_calc_algorithm", "optimized"),
joinqueue=self.join_routes_queue
)
if mode not in ("iv_mitm", "idle"):
coords = self.__fetch_coords(mode, geofence_helper,
coords_spawns_known=area.get("coords_spawns_known", False),
init=area.get("init", False),
range_init=mode_mapping.get(area["mode"], {}).get("range_init", 630),
including_stops=area.get("including_stops", False))
route_manager.add_coords_list(coords)
max_radius = mode_mapping[area["mode"]]["range"]
max_count_in_radius = mode_mapping[area["mode"]]["max_count"]
if not area.get("init", False):
logger.info("Initializing area {}", area["name"])
proc = thread_pool.apply_async(route_manager.recalc_route, args=(max_radius, max_count_in_radius,
0, False))
areas_procs[area["name"]] = proc
else:
logger.info(
"Init mode enabled. Going row-based for {}", str(area.get("name", "unknown")))
# we are in init, let's write the init route to file to make it visible in madmin
if area["routecalc"] is not None:
routefile = os.path.join(
self.__args.file_path, area["routecalc"])
if os.path.isfile(routefile + '.calc'):
os.remove(routefile + '.calc')
with open(routefile + '.calc', 'a') as f:
for loc in coords:
f.write(str(loc.lat) + ', ' +
str(loc.lng) + '\n')
# gotta feed the route to routemanager... TODO: without recalc...
proc = thread_pool.apply_async(route_manager.recalc_route, args=(1, 99999999,
0, False))
areas_procs[area["name"]] = proc
area_dict["routemanager"] = route_manager
areas[area["name"]] = area_dict
for area in areas_procs.keys():
to_be_checked = areas_procs[area]
to_be_checked.get()
thread_pool.close()
thread_pool.join()
return areas
def _generate_locations(distance: float, geofence_helper: GeofenceHelper):
results = []
south, east, north, west = geofence_helper.get_polygon_from_fence()
corners = [
Location(south, east),
Location(south, west),
Location(north, east),
Location(north, west)
]
# get the center
center = get_middle_of_coord_list(corners)
# get the farthest to the center...
farthest_dist = 0
for corner in corners:
dist_temp = get_distance_of_two_points_in_meters(
center.lat, center.lng, corner.lat, corner.lng)
if dist_temp > farthest_dist:
farthest_dist = dist_temp
# calculate step_limit, round up to reduce risk of losing stuff
step_limit = math.ceil(farthest_dist / distance)
# This will loop thorugh all the rings in the hex from the centre
# moving outwards
logger.info("Calculating positions for init scan")
num_cores = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=num_cores) as pool:
temp = [
pool.apply(S2Helper._generate_star_locs,
args=(center, distance, i))
for i in range(1, step_limit)
]
results = [item for sublist in temp for item in sublist]
results.append(Location(center.lat, center.lng))
# for ring in range(1, step_limit):
# for i in range(0, 6):
# # Star_locs will contain the locations of the 6 vertices of
# # the current ring (90,150,210,270,330 and 30 degrees from
# # origin) to form a star
# star_loc = S2Helper.get_new_coords(center, distance * ring,
# 90 + 60 * i)
# for j in range(0, ring):
# # Then from each point on the star, create locations
# # towards the next point of star along the edge of the
# # current ring
# loc = S2Helper.get_new_coords(star_loc, distance * j, 210 + 60 * i)
# results.append(loc)
logger.info("Filtering positions for init scan")
# Geofence results.
if geofence_helper is not None and geofence_helper.is_enabled():
results = geofence_helper.get_geofenced_coordinates(results)
if not results:
logger.error(
'No cells regarded as valid for desired scan area. '
'Check your provided geofences. Aborting.')
sys.exit(1)
logger.info("Ordering location")
results = S2Helper.order_location_list_rows(results)
return results
def get_routemanagers(self, delay_after_hatch=None):
from multiprocessing.pool import ThreadPool
global mode_mapping
# returns list of routemanagers with area IDs
areas = {}
area_arr = self.__raw_json["areas"]
thread_pool = ThreadPool(processes=4)
areas_procs = {}
for area in area_arr:
if area["geofence_included"] is None:
raise RuntimeError("Cannot work without geofence_included")
geofence_included = Path(area["geofence_included"])
if not geofence_included.is_file():
log.error("Geofence included file configured does not exist")
sys.exit(1)
geofence_excluded_raw_path = area.get("geofence_excluded", None)
if geofence_excluded_raw_path is not None:
geofence_excluded = Path(geofence_excluded_raw_path)
if not geofence_excluded.is_file():
log.error("Geofence excluded specified but does not exist")
sys.exit(1)
area_dict = {"mode": area["mode"],
"geofence_included": area["geofence_included"],
"geofence_excluded": area.get("geofence_excluded", None),
"routecalc": area["routecalc"]}
# also build a routemanager for each area...
# grab coords
# first check if init is false or raids_ocr is set as mode, if so, grab the coords from DB
# coords = np.loadtxt(area["coords"], delimiter=',')
geofence_helper = GeofenceHelper(area["geofence_included"], area.get("geofence_excluded", None))
mode = area["mode"]
if mode == "raids_ocr" or area.get("init", False) is False:
# grab data from DB depending on mode
if mode == "raids_ocr" or mode == "raids_mitm":
coords = self.db_wrapper.gyms_from_db(geofence_helper)
elif mode == "mon_mitm":
spawn_known = area.get("coords_spawns_known", False)
if spawn_known:
log.info("Reading known Spawnpoints from DB")
coords = self.db_wrapper.get_detected_spawns(geofence_helper)
else:
log.info("Reading unknown Spawnpoints from DB")
coords = self.db_wrapper.get_undetected_spawns(geofence_helper)
else:
log.fatal("Mode not implemented yet: %s" % str(mode))
exit(1)
else:
# calculate all level N cells (mapping back from mapping above linked to mode)
coords = S2Helper.get_s2_cells_from_fence(geofence=geofence_helper,
cell_size=mode_mapping[mode]["s2_cell_level"])
# coords = S2Helper._generate_locations(mode_mapping[area["mode"]]["range"],
# geofence_helper)
# retrieve the range and max count per circle from central mapping...
route_manager = RouteManager(self.db_wrapper, None, mode_mapping[area["mode"]]["range"],
mode_mapping[area["mode"]]["max_count"],
area["geofence_included"], area.get("geofence_excluded", None),
area["routecalc"],
coords_spawns_known=area.get("coords_spawns_known", False),
delayAfterHatch=delay_after_hatch,
init=area.get("init", False), mode=area["mode"], settings=area["settings"],
name=area.get("name", "unknwon"))
route_manager.add_coords_list(coords)
max_radius = mode_mapping[area["mode"]]["range"]
max_count_in_radius = mode_mapping[area["mode"]]["max_count"]
proc = thread_pool.apply_async(route_manager.recalc_route, args=(max_radius, max_count_in_radius,
0, False))
areas_procs[area["name"]] = proc
# log.error("Calculated route, appending another coord and recalculating")
area_dict["routemanager"] = route_manager
areas[area["name"]] = area_dict
for area in areas_procs.keys():
to_be_checked = areas_procs[area]
log.debug(to_be_checked)
to_be_checked.get()
thread_pool.close()
thread_pool.join()
return areas
from pathlib import Path
from geofence.geofenceHelper import GeofenceHelper
import argparse
log = logging.getLogger(__name__)
parser = argparse.ArgumentParser(description="A program that checks if a lat/lon coord is contained within a geofence")
parser.add_argument("-i", "--include", default="geofence_inc.txt",
help="Geofence include file")
parser.add_argument("-e", "--exclude", default=None,
help="Geofence exclude file")
parser.add_argument("-lat", "--latitude", type=float, required=True,
help="Latitude of coordinate")
parser.add_argument("-lon", "--longitude", type=float, required=True,
help="Longitude of coordinate")
args = parser.parse_args()
if not Path(args.include).is_file():
raise RuntimeError("Geofence included file configured does not exist " + args.include)
if (args.exclude != None ) and (not Path(args.exclude).is_file()):
raise RuntimeError("Geofence excluded file configured does not exist " + args.exclude)
geofence_helper = GeofenceHelper(args.include, args.exclude)
if geofence_helper.is_coord_inside_include_geofence([args.latitude, args.longitude]):
exit(0)
else:
exit(1)
