def __get_installed_version(self): try: self._installed_ver = self.dbwrapper.get_mad_version() if self._installed_ver: logger.info("Internal MAD version in DB is {}", self._installed_ver) else: logger.info( 'Partial schema detected. Additional steps required') self.__install_instance_table() # Attempt to read the old version.json file to get the latest install version in the database try: with open('version.json') as f: self._installed_ver = json.load(f)['version'] logger.success("Moving internal MAD version to database") self.__set_installed_ver(self._installed_ver) except FileNotFoundError: logger.info( 'New installation detected with a partial schema detected. Updates will be attempted' ) self._installed_ver = 0 self.__set_installed_ver(self._installed_ver) self.__reload_instance_id() logger.success( "Moved internal MAD version to database as version {}", self._installed_ver) except Exception: logger.opt(exception=True).critical( 'Unknown exception occurred during getting the MAD DB version.' ' Exiting')
def __apply_update(self, patch_ver): filename = MAD_UPDATES[patch_ver] patch_name = 'mapadroid.patcher.%s' % filename try: patch_base = importlib.import_module(patch_name) except ImportError: logger.opt(exception=True).error( 'Unable to import patch {}. Exiting', patch_name) sys.exit(1) else: # Execute the patch and catch any errors for logging try: patch = patch_base.Patch(logger, self.dbwrapper, self.data_manager, self._application_args) if patch.completed and not patch.issues: self.__set_installed_ver(patch_ver) logger.success('Successfully applied patch') else: logger.error('Patch was unsuccessful. Exiting') sys.exit(1) except Exception: logger.opt( exception=True).error('Patch was unsuccessful. Exiting') sys.exit(1)
def find_latest_pogo(self, architecture: APK_Arch) -> Optional[str]: """ Determine if the package com.nianticlabs.pokemongo has an update Args: architecture (APK_Arch): Architecture of the package to check """ latest = None logger.info('Searching for a new version of PoGo [{}]', architecture.name) self.gpconn = GPlayConnector(architecture) try: download_url = None latest = self.gpconn.get_latest_version(APK_Package.pogo.value) current_version = self.storage.get_current_version( APK_Type.pogo, architecture) if type(current_version) is not str: current_version = None if current_version is None or is_newer_version( latest, current_version): if supported_pogo_version(architecture, latest): logger.info('Newer version found on the Play Store: {}', latest) download_url = True else: logger.info('No newer version found') self.set_last_searched(APK_Type.pogo, architecture, version=latest, url=download_url) except Exception as err: logger.opt(exception=True).critical(err) return latest
def save_file(self, package: APK_Type, architecture: APK_Arch, version: str, mimetype: str, data: BytesIO, retry: bool = False) -> bool: """ Save the package to the filesystem. Remove the old version if it existed Args: package (APK_Type): Package to save architecture (APK_Arch): Architecture of the package to save version (str): Version of the package mimetype (str): Mimetype of the package data (io.BytesIO): binary contents to be saved retry (bool): Attempt to re-save the file if an issue occurs Returns (bool): Save was successful """ try: filename = generate_filename(package, architecture, version, mimetype) with self.file_lock: try: self.delete_file(package, architecture) logger.debug('Successfully removed the previous version') except (FileNotFoundError, KeyError): pass try: self.delete_file(package, architecture) except (TypeError, KeyError): pass try: with open(self.get_package_path(filename), 'wb+') as fh: fh.write(data.getbuffer()) except FileNotFoundError: if retry: self.create_structure() return self.save_file(package, architecture, version, mimetype, data) else: info = { 'version': version, 'file_id': None, 'filename': filename, 'mimetype': mimetype, 'size': os.stat(self.get_package_path(filename)).st_size, } if package not in self.apks: self.apks[package] = MAD_Packages() self.apks[package][architecture] = MAD_Package( package, architecture, **info) self.save_configuration() logger.info('Successfully saved {} to the disk', filename) return True except: # noqa: E722 logger.opt(exception=True).critical('Unable to upload APK') return False
def save_file(self, package: APK_Type, architecture: APK_Arch, version: str, mimetype: str, data: BytesIO, retry: bool = False) -> bool: """ Save the package to the database. Remove the old version if it existed Args: package (APK_Type): Package to save architecture (APK_Arch): Architecture of the package to save version (str): Version of the package mimetype (str): Mimetype of the package data (io.BytesIO): binary contents to be saved retry (bool): Not used Returns (bool): Save was successful """ try: self.delete_file(package, architecture) file_length: int = data.getbuffer().nbytes filename: str = generate_filename(package, architecture, version, mimetype) insert_data = { 'filename': filename, 'size': file_length, 'mimetype': mimetype, } new_id: int = self.dbc.autoexec_insert('filestore_meta', insert_data) insert_data = { 'filestore_id': new_id, 'usage': package.value, 'arch': architecture.value, 'version': version, } self.dbc.autoexec_insert('mad_apks', insert_data, optype='ON DUPLICATE') logger.info('Starting upload of APK') n = global_variables.CHUNK_MAX_SIZE for chunked_data in [ data.getbuffer()[i * n:(i + 1) * n] for i in range((len(data.getbuffer()) + n - 1) // n) ]: insert_data = { 'filestore_id': new_id, 'size': len(chunked_data), 'data': chunked_data.tobytes() } self.dbc.autoexec_insert('filestore_chunks', insert_data) logger.info('Finished upload of APK') return True except: # noqa: E722 logger.opt(exception=True).critical('Unable to upload APK') return False
def run_process(*args, **kwargs): try: run_process_old(*args, **kwargs) except (KeyboardInterrupt, SystemExit): raise except BrokenPipeError: pass except Exception: logger.opt(exception=True).critical("An unhanded exception occurred!")
def internal_error(exception): logger.opt(exception=True).critical("An unhanded exception occurred!") return render_template('500.html'), 500
) mitm_receiver_process.terminate() logger.debug("Trying to join MITMReceiver") mitm_receiver_process.join() logger.debug("MITMReceiver joined") if device_Updater is not None: device_Updater.stop_updater() if t_whw is not None: logger.info("Waiting for webhook-thread to exit") t_whw.join() if ws_server is not None: logger.info("Stopping websocket server") ws_server.stop_server() logger.info("Waiting for websocket-thread to exit") t_ws.join() if mapping_manager_manager is not None: mapping_manager_manager.shutdown() if mitm_mapper_manager is not None: logger.debug("Calling mitm_mapper shutdown") mitm_mapper_manager.shutdown() if db_pool_manager is not None: logger.debug("Calling db_pool_manager shutdown") db_pool_manager.shutdown() logger.debug("Done shutting down db_pool_manager") except: logger.opt(exception=True).critical( "An unhanded exception occurred during shutdown!") logger.info("Done shutting down") logger.debug(str(sys.exc_info())) sys.exit(exit_code)
async def __connection_handler( self, websocket_client_connection: websockets.WebSocketClientProtocol, path: str) -> None: if self.__stop_server.is_set(): return # check auth and stuff TODO origin: Optional[str] = await self.__authenticate_connection( websocket_client_connection) if origin is None: # failed auth, stop connection await self.__close_websocket_client_connection( "Stopping due to failed auth...", websocket_client_connection) return logger.info("New connection with origin {} from {}", origin, websocket_client_connection.remote_address) async with self.__users_connecting_mutex: if origin in self.__users_connecting: logger.info("Client {} is already connecting".format(origin)) return else: self.__users_connecting.add(origin) continue_register = True async with self.__current_users_mutex: logger.debug("Checking if an entry for {} is already present", origin) entry = self.__current_users.get(origin, None) if entry is None: logger.info("Need to start a new worker thread for {}", origin) entry = WebsocketConnectedClientEntry( origin=origin, websocket_client_connection=websocket_client_connection, worker_instance=None, worker_thread=None, loop_running=self.__loop) if not await self.__add_worker_and_thread_to_entry( entry, origin): continue_register = False else: logger.info( "There is a worker thread entry for {} present, handling accordingly", origin) if entry.websocket_client_connection.open: logger.error( "Old connection open while a new one is attempted to be established, " "aborting handling of connection from {}", origin) continue_register = False entry.websocket_client_connection = websocket_client_connection # TODO: also change the worker's Communicator? idk yet if entry.worker_thread.is_alive( ) and not entry.worker_instance.is_stopping(): logger.info( "Worker thread of {} still alive, continue as usual", origin) # TODO: does this need more handling? probably update communicator or whatever? elif not entry.worker_thread.is_alive(): logger.info( "Old thread is dead, trying to start a new one for {}", origin) if not await self.__add_worker_and_thread_to_entry( entry, origin): continue_register = False else: logger.info( "Old thread is about to stop. Wait a little and have {} reconnect", origin) # random sleep to not have clients try again in sync continue_register = False if continue_register: self.__current_users[origin] = entry if not continue_register: await asyncio.sleep(rand.uniform(3, 15)) async with self.__users_connecting_mutex: logger.debug("Removing {} from users_connecting", origin) self.__users_connecting.remove(origin) return try: if not entry.worker_thread.is_alive(): entry.worker_thread.start() # TODO: we need to somehow check threads and synchronize connection status with worker status? async with self.__users_connecting_mutex: self.__users_connecting.remove(origin) receiver_task = asyncio.ensure_future( self.__client_message_receiver(origin, entry)) await receiver_task except Exception as e: logger.opt(exception=True).error( "Other unhandled exception during registration of {}: {}", origin, e) # also check if thread is already running to not start it again. If it is not alive, we need to create it.. finally: logger.info("Awaiting unregister of {}", origin) # TODO: cleanup thread is not really desired, I'd prefer to only restart a worker if the route changes :( self.__worker_shutdown_queue.put(entry.worker_thread) logger.info("Done with connection from {} ({})", origin, websocket_client_connection.remote_address)
async def __register( self, websocket_client_connection: websockets.WebSocketClientProtocol ) -> bool: try: origin = str( websocket_client_connection.request_headers.get_all("Origin") [0]) except IndexError: logger.warning( "Client from {} tried to connect without Origin header", str( websocket_client_connection.request_headers.get_all( "Origin")[0])) return False if not self.__data_manager.is_device_active(origin): logger.warning( 'Origin %s is currently paused. Unpause through MADmin to begin working', origin) return False logger.info("Client {} registering", str(origin)) if self.__mapping_manager is None or origin not in self.__mapping_manager.get_all_devicemappings( ).keys(): logger.warning( "Register attempt of unknown origin: {}. " "Have you forgot to hit 'APPLY SETTINGS' in MADmin?".format( origin)) return False if origin in self.__users_connecting: logger.info("Client {} is already connecting".format(origin)) return False auths = self.__mapping_manager.get_auths() authBase64 = None if auths: try: authBase64 = str( websocket_client_connection.request_headers.get_all( "Authorization")[0]) except IndexError: logger.warning( "Client from {} tried to connect without auth header", str( websocket_client_connection.request_headers.get_all( "Origin")[0])) return False worker_already_connected: bool = False async with self.__users_mutex: logger.debug("Checking if {} is already present", str(origin)) if origin in self.__current_users: logger.warning( "Worker with origin {} is already running, killing the running one and have client reconnect", str(origin)) self.__current_users.get(origin)[1].stop_worker() worker_already_connected = True else: self.__users_connecting.append(origin) if worker_already_connected: ## todo: do this better :D logger.debug( "Old worker thread is still alive - waiting 20 seconds") await asyncio.sleep(20) logger.info("Reconnect ...") return False # reset pref. error counter if exist await self.__reset_fail_counter(origin) try: if auths and authBase64 and not check_auth(authBase64, self.args, auths): logger.warning( "Invalid auth details received from {}", str( websocket_client_connection.request_headers.get_all( "Origin")[0])) return False logger.info("Starting worker {}".format(origin)) if self._configmode: dev_id = self.__mapping_manager.get_all_devicemappings( )[origin]['device_id'] devicesettings = self.__mapping_manager.get_devicesettings_of( origin) client_mapping = self.__mapping_manager.get_devicemappings_of( origin) walker_area_array = client_mapping["walker"] walker_index = devicesettings.get('walker_area_index', 0) walker_settings = walker_area_array[walker_index] area_id = walker_settings['walkerarea'] worker = WorkerConfigmode( args=self.args, dev_id=dev_id, id=origin, websocket_handler=self, walker=None, mapping_manager=self.__mapping_manager, mitm_mapper=self.__mitm_mapper, db_wrapper=self.__db_wrapper, area_id=area_id, routemanager_name=None) logger.debug("Starting worker for {}", str(origin)) new_worker_thread = Thread(name='worker_%s' % origin, target=worker.start_worker) async with self.__users_mutex: self.__current_users[origin] = [ new_worker_thread, worker, websocket_client_connection, 0 ] return True last_known_state = {} client_mapping = self.__mapping_manager.get_devicemappings_of( origin) devicesettings = self.__mapping_manager.get_devicesettings_of( origin) logger.info("Setting up routemanagers for {}", str(origin)) if client_mapping.get("walker", None) is not None: if devicesettings is not None and "walker_area_index" not in devicesettings: logger.debug("Initializing devicesettings") self.__mapping_manager.set_devicesetting_value_of( origin, 'walker_area_index', 0) self.__mapping_manager.set_devicesetting_value_of( origin, 'finished', False) self.__mapping_manager.set_devicesetting_value_of( origin, 'last_action_time', None) self.__mapping_manager.set_devicesetting_value_of( origin, 'last_cleanup_time', None) self.__mapping_manager.set_devicesetting_value_of( origin, 'job', False) await asyncio.sleep( 1 ) # give the settings a moment... (dirty "workaround" against race condition) walker_index = devicesettings.get('walker_area_index', 0) if walker_index > 0: # check status of last area if not devicesettings.get('finished', False): logger.info( 'Something wrong with last round - get back to old area' ) walker_index -= 1 self.__mapping_manager.set_devicesetting_value_of( origin, 'walker_area_index', walker_index) # devicesettings['walker_area_index'] = walker_index walker_area_array = client_mapping["walker"] walker_settings = walker_area_array[walker_index] # preckeck walker setting while not pre_check_value( walker_settings) and walker_index - 1 <= len( walker_area_array): walker_area_name = walker_area_array[walker_index][ 'walkerarea'] logger.info( '{} not using area {} - Walkervalue out of range', str(origin), str( self.__mapping_manager.routemanager_get_name( walker_area_name))) if walker_index >= len(walker_area_array) - 1: logger.error( 'Could not find any working area at this time - check your mappings for device: {}', str(origin)) walker_index = 0 self.__mapping_manager.set_devicesetting_value_of( origin, 'walker_area_index', walker_index) walker_settings = walker_area_array[walker_index] return False walker_index += 1 self.__mapping_manager.set_devicesetting_value_of( origin, 'walker_area_index', walker_index) walker_settings = walker_area_array[walker_index] devicesettings = self.__mapping_manager.get_devicesettings_of( origin) logger.debug("Checking walker_area_index length") if (devicesettings.get("walker_area_index", None) is None or devicesettings['walker_area_index'] >= len(walker_area_array)): # check if array is smaller than expected - f.e. on the fly changes in mappings.json self.__mapping_manager.set_devicesetting_value_of( origin, 'walker_area_index', 0) self.__mapping_manager.set_devicesetting_value_of( origin, 'finished', False) walker_index = 0 walker_area_name = walker_area_array[walker_index][ 'walkerarea'] if walker_area_name not in self.__mapping_manager.get_all_routemanager_names( ): raise WrongAreaInWalker() logger.debug('Devicesettings {}: {}', str(origin), devicesettings) logger.info( '{} using walker area {} [{}/{}]', str(origin), str( self.__mapping_manager.routemanager_get_name( walker_area_name)), str(walker_index + 1), str(len(walker_area_array))) walker_routemanager_mode = self.__mapping_manager.routemanager_get_mode( walker_area_name) self.__mapping_manager.set_devicesetting_value_of( origin, 'walker_area_index', walker_index + 1) self.__mapping_manager.set_devicesetting_value_of( origin, 'finished', False) if walker_index >= len(walker_area_array) - 1: self.__mapping_manager.set_devicesetting_value_of( origin, 'walker_area_index', 0) # set global mon_iv routemanager_settings = self.__mapping_manager.routemanager_get_settings( walker_area_name) if routemanager_settings is not None: client_mapping['mon_ids_iv'] = \ self.__mapping_manager.get_monlist(routemanager_settings.get("mon_ids_iv", None), walker_area_name) else: walker_routemanager_mode = None if "last_location" not in devicesettings: devicesettings['last_location'] = Location(0.0, 0.0) logger.debug("Setting up worker for {}", str(origin)) dev_id = self.__mapping_manager.get_all_devicemappings( )[origin]['device_id'] area_id = walker_settings['walkerarea'] worker = None if walker_routemanager_mode is None: pass elif walker_routemanager_mode in [ "raids_mitm", "mon_mitm", "iv_mitm" ]: worker = WorkerMITM( self.args, dev_id, origin, last_known_state, self, area_id=area_id, routemanager_name=walker_area_name, mitm_mapper=self.__mitm_mapper, mapping_manager=self.__mapping_manager, db_wrapper=self.__db_wrapper, pogo_window_manager=self.__pogoWindowManager, walker=walker_settings) elif walker_routemanager_mode in ["pokestops"]: worker = WorkerQuests( self.args, dev_id, origin, last_known_state, self, area_id=area_id, routemanager_name=walker_area_name, mitm_mapper=self.__mitm_mapper, mapping_manager=self.__mapping_manager, db_wrapper=self.__db_wrapper, pogo_window_manager=self.__pogoWindowManager, walker=walker_settings) elif walker_routemanager_mode in ["idle"]: worker = WorkerConfigmode( self.args, dev_id, origin, self, walker=walker_settings, mapping_manager=self.__mapping_manager, mitm_mapper=self.__mitm_mapper, db_wrapper=self.__db_wrapper, area_id=area_id, routemanager_name=walker_area_name) else: logger.error("Mode not implemented") sys.exit(1) if worker is None: logger.error( "Invalid walker mode for {}. Closing connection".format( str(origin))) return False else: logger.debug("Starting worker for {}", str(origin)) new_worker_thread = Thread(name='worker_%s' % origin, target=worker.start_worker) new_worker_thread.daemon = True async with self.__users_mutex: self.__current_users[origin] = [ new_worker_thread, worker, websocket_client_connection, 0 ] new_worker_thread.start() except WrongAreaInWalker: logger.error('Unknown Area in Walker settings - check config') return False except Exception as e: logger.opt(exception=True).error( "Other unhandled exception during registration of {}: {}", origin, e) return False finally: async with self.__users_mutex: self.__users_connecting.remove(origin) await asyncio.sleep(5) logger.info("Done handling register of origin ", origin) return True