def __get_installed_version(self):
try:
self._installed_ver = self.dbwrapper.get_mad_version()
if self._installed_ver:
logger.info("Internal MAD version in DB is {}",
self._installed_ver)
else:
logger.info(
'Partial schema detected. Additional steps required')
self.__install_instance_table()
# Attempt to read the old version.json file to get the latest install version in the database
try:
with open('version.json') as f:
self._installed_ver = json.load(f)['version']
logger.success("Moving internal MAD version to database")
self.__set_installed_ver(self._installed_ver)
except FileNotFoundError:
logger.info(
'New installation detected with a partial schema detected. Updates will be attempted'
)
self._installed_ver = 0
self.__set_installed_ver(self._installed_ver)
self.__reload_instance_id()
logger.success(
"Moved internal MAD version to database as version {}",
self._installed_ver)
except Exception:
logger.opt(exception=True).critical(
'Unknown exception occurred during getting the MAD DB version.'
' Exiting')
def __apply_update(self, patch_ver):
filename = MAD_UPDATES[patch_ver]
patch_name = 'mapadroid.patcher.%s' % filename
try:
patch_base = importlib.import_module(patch_name)
except ImportError:
logger.opt(exception=True).error(
'Unable to import patch {}. Exiting', patch_name)
sys.exit(1)
else:
# Execute the patch and catch any errors for logging
try:
patch = patch_base.Patch(logger, self.dbwrapper,
self.data_manager,
self._application_args)
if patch.completed and not patch.issues:
self.__set_installed_ver(patch_ver)
logger.success('Successfully applied patch')
else:
logger.error('Patch was unsuccessful. Exiting')
sys.exit(1)
except Exception:
logger.opt(
exception=True).error('Patch was unsuccessful. Exiting')
sys.exit(1)
def find_latest_pogo(self, architecture: APK_Arch) -> Optional[str]:
""" Determine if the package com.nianticlabs.pokemongo has an update
Args:
architecture (APK_Arch): Architecture of the package to check
"""
latest = None
logger.info('Searching for a new version of PoGo [{}]',
architecture.name)
self.gpconn = GPlayConnector(architecture)
try:
download_url = None
latest = self.gpconn.get_latest_version(APK_Package.pogo.value)
current_version = self.storage.get_current_version(
APK_Type.pogo, architecture)
if type(current_version) is not str:
current_version = None
if current_version is None or is_newer_version(
latest, current_version):
if supported_pogo_version(architecture, latest):
logger.info('Newer version found on the Play Store: {}',
latest)
download_url = True
else:
logger.info('No newer version found')
self.set_last_searched(APK_Type.pogo,
architecture,
version=latest,
url=download_url)
except Exception as err:
logger.opt(exception=True).critical(err)
return latest
def save_file(self,
package: APK_Type,
architecture: APK_Arch,
version: str,
mimetype: str,
data: BytesIO,
retry: bool = False) -> bool:
""" Save the package to the filesystem. Remove the old version if it existed
Args:
package (APK_Type): Package to save
architecture (APK_Arch): Architecture of the package to save
version (str): Version of the package
mimetype (str): Mimetype of the package
data (io.BytesIO): binary contents to be saved
retry (bool): Attempt to re-save the file if an issue occurs
Returns (bool):
Save was successful
"""
try:
filename = generate_filename(package, architecture, version,
mimetype)
with self.file_lock:
try:
self.delete_file(package, architecture)
logger.debug('Successfully removed the previous version')
except (FileNotFoundError, KeyError):
pass
try:
self.delete_file(package, architecture)
except (TypeError, KeyError):
pass
try:
with open(self.get_package_path(filename), 'wb+') as fh:
fh.write(data.getbuffer())
except FileNotFoundError:
if retry:
self.create_structure()
return self.save_file(package, architecture, version,
mimetype, data)
else:
info = {
'version': version,
'file_id': None,
'filename': filename,
'mimetype': mimetype,
'size':
os.stat(self.get_package_path(filename)).st_size,
}
if package not in self.apks:
self.apks[package] = MAD_Packages()
self.apks[package][architecture] = MAD_Package(
package, architecture, **info)
self.save_configuration()
logger.info('Successfully saved {} to the disk', filename)
return True
except: # noqa: E722
logger.opt(exception=True).critical('Unable to upload APK')
return False
def save_file(self,
package: APK_Type,
architecture: APK_Arch,
version: str,
mimetype: str,
data: BytesIO,
retry: bool = False) -> bool:
""" Save the package to the database. Remove the old version if it existed
Args:
package (APK_Type): Package to save
architecture (APK_Arch): Architecture of the package to save
version (str): Version of the package
mimetype (str): Mimetype of the package
data (io.BytesIO): binary contents to be saved
retry (bool): Not used
Returns (bool):
Save was successful
"""
try:
self.delete_file(package, architecture)
file_length: int = data.getbuffer().nbytes
filename: str = generate_filename(package, architecture, version,
mimetype)
insert_data = {
'filename': filename,
'size': file_length,
'mimetype': mimetype,
}
new_id: int = self.dbc.autoexec_insert('filestore_meta',
insert_data)
insert_data = {
'filestore_id': new_id,
'usage': package.value,
'arch': architecture.value,
'version': version,
}
self.dbc.autoexec_insert('mad_apks',
insert_data,
optype='ON DUPLICATE')
logger.info('Starting upload of APK')
n = global_variables.CHUNK_MAX_SIZE
for chunked_data in [
data.getbuffer()[i * n:(i + 1) * n]
for i in range((len(data.getbuffer()) + n - 1) // n)
]:
insert_data = {
'filestore_id': new_id,
'size': len(chunked_data),
'data': chunked_data.tobytes()
}
self.dbc.autoexec_insert('filestore_chunks', insert_data)
logger.info('Finished upload of APK')
return True
except: # noqa: E722
logger.opt(exception=True).critical('Unable to upload APK')
return False
def run_process(*args, **kwargs):
try:
run_process_old(*args, **kwargs)
except (KeyboardInterrupt, SystemExit):
raise
except BrokenPipeError:
pass
except Exception:
logger.opt(exception=True).critical("An unhanded exception occurred!")
def internal_error(exception):
logger.opt(exception=True).critical("An unhanded exception occurred!")
return render_template('500.html'), 500
)
mitm_receiver_process.terminate()
logger.debug("Trying to join MITMReceiver")
mitm_receiver_process.join()
logger.debug("MITMReceiver joined")
if device_Updater is not None:
device_Updater.stop_updater()
if t_whw is not None:
logger.info("Waiting for webhook-thread to exit")
t_whw.join()
if ws_server is not None:
logger.info("Stopping websocket server")
ws_server.stop_server()
logger.info("Waiting for websocket-thread to exit")
t_ws.join()
if mapping_manager_manager is not None:
mapping_manager_manager.shutdown()
if mitm_mapper_manager is not None:
logger.debug("Calling mitm_mapper shutdown")
mitm_mapper_manager.shutdown()
if db_pool_manager is not None:
logger.debug("Calling db_pool_manager shutdown")
db_pool_manager.shutdown()
logger.debug("Done shutting down db_pool_manager")
except:
logger.opt(exception=True).critical(
"An unhanded exception occurred during shutdown!")
logger.info("Done shutting down")
logger.debug(str(sys.exc_info()))
sys.exit(exit_code)
async def __connection_handler(
self,
websocket_client_connection: websockets.WebSocketClientProtocol,
path: str) -> None:
if self.__stop_server.is_set():
return
# check auth and stuff TODO
origin: Optional[str] = await self.__authenticate_connection(
websocket_client_connection)
if origin is None:
# failed auth, stop connection
await self.__close_websocket_client_connection(
"Stopping due to failed auth...", websocket_client_connection)
return
logger.info("New connection with origin {} from {}", origin,
websocket_client_connection.remote_address)
async with self.__users_connecting_mutex:
if origin in self.__users_connecting:
logger.info("Client {} is already connecting".format(origin))
return
else:
self.__users_connecting.add(origin)
continue_register = True
async with self.__current_users_mutex:
logger.debug("Checking if an entry for {} is already present",
origin)
entry = self.__current_users.get(origin, None)
if entry is None:
logger.info("Need to start a new worker thread for {}", origin)
entry = WebsocketConnectedClientEntry(
origin=origin,
websocket_client_connection=websocket_client_connection,
worker_instance=None,
worker_thread=None,
loop_running=self.__loop)
if not await self.__add_worker_and_thread_to_entry(
entry, origin):
continue_register = False
else:
logger.info(
"There is a worker thread entry for {} present, handling accordingly",
origin)
if entry.websocket_client_connection.open:
logger.error(
"Old connection open while a new one is attempted to be established, "
"aborting handling of connection from {}", origin)
continue_register = False
entry.websocket_client_connection = websocket_client_connection
# TODO: also change the worker's Communicator? idk yet
if entry.worker_thread.is_alive(
) and not entry.worker_instance.is_stopping():
logger.info(
"Worker thread of {} still alive, continue as usual",
origin)
# TODO: does this need more handling? probably update communicator or whatever?
elif not entry.worker_thread.is_alive():
logger.info(
"Old thread is dead, trying to start a new one for {}",
origin)
if not await self.__add_worker_and_thread_to_entry(
entry, origin):
continue_register = False
else:
logger.info(
"Old thread is about to stop. Wait a little and have {} reconnect",
origin)
# random sleep to not have clients try again in sync
continue_register = False
if continue_register:
self.__current_users[origin] = entry
if not continue_register:
await asyncio.sleep(rand.uniform(3, 15))
async with self.__users_connecting_mutex:
logger.debug("Removing {} from users_connecting", origin)
self.__users_connecting.remove(origin)
return
try:
if not entry.worker_thread.is_alive():
entry.worker_thread.start()
# TODO: we need to somehow check threads and synchronize connection status with worker status?
async with self.__users_connecting_mutex:
self.__users_connecting.remove(origin)
receiver_task = asyncio.ensure_future(
self.__client_message_receiver(origin, entry))
await receiver_task
except Exception as e:
logger.opt(exception=True).error(
"Other unhandled exception during registration of {}: {}",
origin, e)
# also check if thread is already running to not start it again. If it is not alive, we need to create it..
finally:
logger.info("Awaiting unregister of {}", origin)
# TODO: cleanup thread is not really desired, I'd prefer to only restart a worker if the route changes :(
self.__worker_shutdown_queue.put(entry.worker_thread)
logger.info("Done with connection from {} ({})", origin,
websocket_client_connection.remote_address)
async def __register(
self, websocket_client_connection: websockets.WebSocketClientProtocol
) -> bool:
try:
origin = str(
websocket_client_connection.request_headers.get_all("Origin")
[0])
except IndexError:
logger.warning(
"Client from {} tried to connect without Origin header",
str(
websocket_client_connection.request_headers.get_all(
"Origin")[0]))
return False
if not self.__data_manager.is_device_active(origin):
logger.warning(
'Origin %s is currently paused. Unpause through MADmin to begin working',
origin)
return False
logger.info("Client {} registering", str(origin))
if self.__mapping_manager is None or origin not in self.__mapping_manager.get_all_devicemappings(
).keys():
logger.warning(
"Register attempt of unknown origin: {}. "
"Have you forgot to hit 'APPLY SETTINGS' in MADmin?".format(
origin))
return False
if origin in self.__users_connecting:
logger.info("Client {} is already connecting".format(origin))
return False
auths = self.__mapping_manager.get_auths()
authBase64 = None
if auths:
try:
authBase64 = str(
websocket_client_connection.request_headers.get_all(
"Authorization")[0])
except IndexError:
logger.warning(
"Client from {} tried to connect without auth header",
str(
websocket_client_connection.request_headers.get_all(
"Origin")[0]))
return False
worker_already_connected: bool = False
async with self.__users_mutex:
logger.debug("Checking if {} is already present", str(origin))
if origin in self.__current_users:
logger.warning(
"Worker with origin {} is already running, killing the running one and have client reconnect",
str(origin))
self.__current_users.get(origin)[1].stop_worker()
worker_already_connected = True
else:
self.__users_connecting.append(origin)
if worker_already_connected:
## todo: do this better :D
logger.debug(
"Old worker thread is still alive - waiting 20 seconds")
await asyncio.sleep(20)
logger.info("Reconnect ...")
return False
# reset pref. error counter if exist
await self.__reset_fail_counter(origin)
try:
if auths and authBase64 and not check_auth(authBase64, self.args,
auths):
logger.warning(
"Invalid auth details received from {}",
str(
websocket_client_connection.request_headers.get_all(
"Origin")[0]))
return False
logger.info("Starting worker {}".format(origin))
if self._configmode:
dev_id = self.__mapping_manager.get_all_devicemappings(
)[origin]['device_id']
devicesettings = self.__mapping_manager.get_devicesettings_of(
origin)
client_mapping = self.__mapping_manager.get_devicemappings_of(
origin)
walker_area_array = client_mapping["walker"]
walker_index = devicesettings.get('walker_area_index', 0)
walker_settings = walker_area_array[walker_index]
area_id = walker_settings['walkerarea']
worker = WorkerConfigmode(
args=self.args,
dev_id=dev_id,
id=origin,
websocket_handler=self,
walker=None,
mapping_manager=self.__mapping_manager,
mitm_mapper=self.__mitm_mapper,
db_wrapper=self.__db_wrapper,
area_id=area_id,
routemanager_name=None)
logger.debug("Starting worker for {}", str(origin))
new_worker_thread = Thread(name='worker_%s' % origin,
target=worker.start_worker)
async with self.__users_mutex:
self.__current_users[origin] = [
new_worker_thread, worker, websocket_client_connection,
0
]
return True
last_known_state = {}
client_mapping = self.__mapping_manager.get_devicemappings_of(
origin)
devicesettings = self.__mapping_manager.get_devicesettings_of(
origin)
logger.info("Setting up routemanagers for {}", str(origin))
if client_mapping.get("walker", None) is not None:
if devicesettings is not None and "walker_area_index" not in devicesettings:
logger.debug("Initializing devicesettings")
self.__mapping_manager.set_devicesetting_value_of(
origin, 'walker_area_index', 0)
self.__mapping_manager.set_devicesetting_value_of(
origin, 'finished', False)
self.__mapping_manager.set_devicesetting_value_of(
origin, 'last_action_time', None)
self.__mapping_manager.set_devicesetting_value_of(
origin, 'last_cleanup_time', None)
self.__mapping_manager.set_devicesetting_value_of(
origin, 'job', False)
await asyncio.sleep(
1
) # give the settings a moment... (dirty "workaround" against race condition)
walker_index = devicesettings.get('walker_area_index', 0)
if walker_index > 0:
# check status of last area
if not devicesettings.get('finished', False):
logger.info(
'Something wrong with last round - get back to old area'
)
walker_index -= 1
self.__mapping_manager.set_devicesetting_value_of(
origin, 'walker_area_index', walker_index)
# devicesettings['walker_area_index'] = walker_index
walker_area_array = client_mapping["walker"]
walker_settings = walker_area_array[walker_index]
# preckeck walker setting
while not pre_check_value(
walker_settings) and walker_index - 1 <= len(
walker_area_array):
walker_area_name = walker_area_array[walker_index][
'walkerarea']
logger.info(
'{} not using area {} - Walkervalue out of range',
str(origin),
str(
self.__mapping_manager.routemanager_get_name(
walker_area_name)))
if walker_index >= len(walker_area_array) - 1:
logger.error(
'Could not find any working area at this time - check your mappings for device: {}',
str(origin))
walker_index = 0
self.__mapping_manager.set_devicesetting_value_of(
origin, 'walker_area_index', walker_index)
walker_settings = walker_area_array[walker_index]
return False
walker_index += 1
self.__mapping_manager.set_devicesetting_value_of(
origin, 'walker_area_index', walker_index)
walker_settings = walker_area_array[walker_index]
devicesettings = self.__mapping_manager.get_devicesettings_of(
origin)
logger.debug("Checking walker_area_index length")
if (devicesettings.get("walker_area_index", None) is None
or devicesettings['walker_area_index'] >=
len(walker_area_array)):
# check if array is smaller than expected - f.e. on the fly changes in mappings.json
self.__mapping_manager.set_devicesetting_value_of(
origin, 'walker_area_index', 0)
self.__mapping_manager.set_devicesetting_value_of(
origin, 'finished', False)
walker_index = 0
walker_area_name = walker_area_array[walker_index][
'walkerarea']
if walker_area_name not in self.__mapping_manager.get_all_routemanager_names(
):
raise WrongAreaInWalker()
logger.debug('Devicesettings {}: {}', str(origin),
devicesettings)
logger.info(
'{} using walker area {} [{}/{}]', str(origin),
str(
self.__mapping_manager.routemanager_get_name(
walker_area_name)), str(walker_index + 1),
str(len(walker_area_array)))
walker_routemanager_mode = self.__mapping_manager.routemanager_get_mode(
walker_area_name)
self.__mapping_manager.set_devicesetting_value_of(
origin, 'walker_area_index', walker_index + 1)
self.__mapping_manager.set_devicesetting_value_of(
origin, 'finished', False)
if walker_index >= len(walker_area_array) - 1:
self.__mapping_manager.set_devicesetting_value_of(
origin, 'walker_area_index', 0)
# set global mon_iv
routemanager_settings = self.__mapping_manager.routemanager_get_settings(
walker_area_name)
if routemanager_settings is not None:
client_mapping['mon_ids_iv'] = \
self.__mapping_manager.get_monlist(routemanager_settings.get("mon_ids_iv", None),
walker_area_name)
else:
walker_routemanager_mode = None
if "last_location" not in devicesettings:
devicesettings['last_location'] = Location(0.0, 0.0)
logger.debug("Setting up worker for {}", str(origin))
dev_id = self.__mapping_manager.get_all_devicemappings(
)[origin]['device_id']
area_id = walker_settings['walkerarea']
worker = None
if walker_routemanager_mode is None:
pass
elif walker_routemanager_mode in [
"raids_mitm", "mon_mitm", "iv_mitm"
]:
worker = WorkerMITM(
self.args,
dev_id,
origin,
last_known_state,
self,
area_id=area_id,
routemanager_name=walker_area_name,
mitm_mapper=self.__mitm_mapper,
mapping_manager=self.__mapping_manager,
db_wrapper=self.__db_wrapper,
pogo_window_manager=self.__pogoWindowManager,
walker=walker_settings)
elif walker_routemanager_mode in ["pokestops"]:
worker = WorkerQuests(
self.args,
dev_id,
origin,
last_known_state,
self,
area_id=area_id,
routemanager_name=walker_area_name,
mitm_mapper=self.__mitm_mapper,
mapping_manager=self.__mapping_manager,
db_wrapper=self.__db_wrapper,
pogo_window_manager=self.__pogoWindowManager,
walker=walker_settings)
elif walker_routemanager_mode in ["idle"]:
worker = WorkerConfigmode(
self.args,
dev_id,
origin,
self,
walker=walker_settings,
mapping_manager=self.__mapping_manager,
mitm_mapper=self.__mitm_mapper,
db_wrapper=self.__db_wrapper,
area_id=area_id,
routemanager_name=walker_area_name)
else:
logger.error("Mode not implemented")
sys.exit(1)
if worker is None:
logger.error(
"Invalid walker mode for {}. Closing connection".format(
str(origin)))
return False
else:
logger.debug("Starting worker for {}", str(origin))
new_worker_thread = Thread(name='worker_%s' % origin,
target=worker.start_worker)
new_worker_thread.daemon = True
async with self.__users_mutex:
self.__current_users[origin] = [
new_worker_thread, worker, websocket_client_connection,
0
]
new_worker_thread.start()
except WrongAreaInWalker:
logger.error('Unknown Area in Walker settings - check config')
return False
except Exception as e:
logger.opt(exception=True).error(
"Other unhandled exception during registration of {}: {}",
origin, e)
return False
finally:
async with self.__users_mutex:
self.__users_connecting.remove(origin)
await asyncio.sleep(5)
logger.info("Done handling register of origin ", origin)
return True
