def get_nest_points(): nests = [] try: nests = load_pickle('nests', raise_exception=True) except (FileNotFoundError, TypeError, KeyError): parks = get_all_parks() for s in get_spawnpoint_markers(): spawn_point = Point(s['lat'], s['lon']) cell = Polygon(get_s2_cell_as_polygon(s['lat'], s['lon'], 20)) # s2 lvl 20 for p in parks: coords = p['coords'] # osm polygon can be a line if len(coords) == 2: shape = LineString(coords) if shape.within(cell.centroid): nests.append(s) break if len(coords) > 2: shape = Polygon(coords) if shape.contains(cell.centroid): nests.append(s) break dump_pickle('nests', nests) return nests
def cleanup(overseer, manager): try: overseer.print_handle.cancel() overseer.running = False print('Exiting, please wait until all tasks finish') log = get_logger('cleanup') print('Finishing tasks...') LOOP.create_task(overseer.exit_progress()) pending = gather(*Task.all_tasks(loop=LOOP), return_exceptions=True) try: LOOP.run_until_complete(wait_for(pending, 40)) except TimeoutError as e: print('Coroutine completion timed out, moving on.') except Exception as e: log = get_logger('cleanup') log.exception('A wild {} appeared during exit!', e.__class__.__name__) db_proc.stop() overseer.refresh_dict() print('Dumping pickles...') dump_pickle('accounts', ACCOUNTS) FORT_CACHE.pickle() RAID_CACHE.pickle() altitudes.pickle() if conf.CACHE_CELLS: dump_pickle('cells', Worker.cells) spawns.pickle() while not db_proc.queue.empty(): pending = db_proc.queue.qsize() # Spaces at the end are important, as they clear previously printed # output - \r doesn't clean whole line print('{} DB items pending '.format(pending), end='\r') sleep(.5) finally: print('Closing pipes, sessions, and event loop...') manager.shutdown() SessionManager.close() close_sessions() LOOP.close() print('Done.')
def get_all_parks(): parks = [] try: parks = load_pickle('parks', raise_exception=True) except (FileNotFoundError, TypeError, KeyError): # all osm parks at 07/17/2016 api = overpy.Overpass() request = '[timeout:620][date:"2016-07-17T00:00:00Z"];(way["leisure"="park"];way["landuse"="recreation_ground"];way["leisure"="recreation_ground"];way["leisure"="pitch"];way["leisure"="garden"];way["leisure"="golf_course"];way["leisure"="playground"];way["landuse"="meadow"];way["landuse"="grass"];way["landuse"="greenfield"];way["natural"="scrub"];way["natural"="heath"];way["natural"="grassland"];way["landuse"="farmyard"];way["landuse"="vineyard"];way["natural"="plateau"];way["leisure"="nature_reserve"];way["natural"="moor"];way["landuse"="farmland"];way["landuse"="orchard"];);out;>;out skel qt;' request = '[bbox:{},{},{},{}]{}'.format(south, west, north, east, request) response = api.query(request) for w in response.ways: parks.append({ 'type': 'park', 'coords': [[float(c.lat), float(c.lon)] for c in w.nodes] }) dump_pickle('parks', parks) return parks
def cleanup(overseer, manager): try: overseer.print_handle.cancel() overseer.running = False print('Exiting, please wait until all tasks finish') log = get_logger('cleanup') print('Finishing tasks...') LOOP.create_task(overseer.exit_progress()) pending = gather(*Task.all_tasks(loop=LOOP), return_exceptions=True) try: LOOP.run_until_complete(wait_for(pending, 40)) except TimeoutError as e: print('Coroutine completion timed out, moving on.') except Exception as e: log = get_logger('cleanup') log.exception('A wild {} appeared during exit!', e.__class__.__name__) db_proc.stop() overseer.refresh_dict() print('Dumping pickles...') dump_pickle('accounts', ACCOUNTS) FORT_CACHE.pickle() altitudes.pickle() if conf.CACHE_CELLS: dump_pickle('cells', Worker.cells) spawns.pickle() while not db_proc.queue.empty(): pending = db_proc.queue.qsize() # Spaces at the end are important, as they clear previously printed # output - \r doesn't clean whole line print('{} DB items pending '.format(pending), end='\r') sleep(.5) finally: print('Closing pipes, sessions, and event loop...') manager.shutdown() SessionManager.close() close_sessions() LOOP.close() print('Done.')
def main(): args = parse_args() logger = getLogger() if args.status_bar: configure_logger(filename=join(config.DIRECTORY, 'scan.log')) logger.info('-' * 30) logger.info('Starting up!') else: configure_logger(filename=None) logger.setLevel(args.log_level) AccountManager.register('captcha_queue', callable=get_captchas) AccountManager.register('extra_queue', callable=get_extras) if config.MAP_WORKERS: AccountManager.register('worker_dict', callable=get_workers, proxytype=DictProxy) address = get_address() manager = AccountManager(address=address, authkey=config.AUTHKEY) try: manager.start(mgr_init) except (OSError, EOFError) as e: if platform == 'win32' or not isinstance(address, str): raise OSError('Another instance is running with the same manager address. Stop that process or change your MANAGER_ADDRESS.') from e else: raise OSError('Another instance is running with the same socket. Stop that process or: rm {}'.format(address)) from e loop = asyncio.get_event_loop() loop.set_exception_handler(exception_handler) overseer = Overseer(status_bar=args.status_bar, manager=manager) overseer.start() overseer_thread = Thread(target=overseer.check, name='overseer', daemon=True) overseer_thread.start() launcher_thread = Thread(target=overseer.launch, name='launcher', daemon=True, args=(args.bootstrap, args.pickle)) launcher_thread.start() try: loop.run_forever() except KeyboardInterrupt: print('Exiting, please wait until all tasks finish') overseer.kill() dump_pickle('accounts', Worker.accounts) if config.CACHE_CELLS: dump_pickle('cells', Worker.cell_ids) pending = asyncio.Task.all_tasks(loop=loop) try: loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) except Exception: logger.exception('A wild exception appeared during exit!') Worker.db_processor.stop() try: Worker.spawns.update() except (DBAPIError): pass Worker.spawns.session.close() manager.shutdown() Session.close() try: loop.close() except RuntimeError: pass
def get_ex_gyms(): ex_gyms = [] parks = get_all_parks() try: ex_gyms = load_pickle('ex_gyms', raise_exception=True) except (FileNotFoundError, TypeError, KeyError): for g in get_gym_markers(): g['id'] = 'ex-' + g['id'] gym_point = Point(g['lat'], g['lon']) cell = Polygon(get_s2_cell_as_polygon(g['lat'], g['lon'], level=20)) # s2 lvl 20 sponsor_value = 'park' if g['sponsor'] == 7: sponsor_value = 'starbucks' ex_gyms.append({ 'id': g['id'], 'external_id': g['external_id'], 'fort_id': g['fort_id'], 'name': g['gym_name'], 'lat': g['lat'], 'lon': g['lon'], 'sponsor': sponsor_value }) if g['sponsor'] == 9: sponsor_value = 'sprint' ex_gyms.append({ 'id': g['id'], 'external_id': g['external_id'], 'fort_id': g['fort_id'], 'name': g['gym_name'], 'lat': g['lat'], 'lon': g['lon'], 'sponsor': sponsor_value }) for p in parks: coords = p['coords'] # osm polygon can be a line if len(coords) == 2: shape = LineString(coords) if shape.within(cell.centroid): ex_gyms.append({ 'id': g['id'], 'external_id': g['external_id'], 'fort_id': g['fort_id'], 'name': g['gym_name'], 'lat': g['lat'], 'lon': g['lon'], 'sponsor': sponsor_value }) if len(coords) > 2: shape = Polygon(coords) if shape.contains(cell.centroid): ex_gyms.append({ 'id': g['id'], 'external_id': g['external_id'], 'fort_id': g['fort_id'], 'name': g['gym_name'], 'lat': g['lat'], 'lon': g['lon'], 'sponsor': sponsor_value }) dump_pickle('ex_gyms', ex_gyms) return ex_gyms