def received_data_worker(self, multi_proc_queue: Queue, application_args,
mitm_mapper: MitmMapper):
# build a private DbWrapper instance...
self.__queue: Queue = multi_proc_queue
self.__db_wrapper: DbWrapperBase = DbFactory.get_wrapper(
application_args)
self.__application_args = application_args
self.__mitm_mapper: MitmMapper = mitm_mapper
while True:
try:
item = self.__queue.get()
except KeyboardInterrupt as e:
logger.info(
"MITMDataProcessor received keyboard interrupt, stopping")
break
items_left = self.__queue.qsize()
logger.debug(
"MITM data processing worker retrieved data. Queue length left afterwards: {}",
str(items_left))
if items_left > 50:
logger.warning(
"MITM data processing workers are falling behind! Queue length: {}",
str(items_left))
if item is None:
logger.warning("Received none from queue of data")
break
self.process_data(item[0], item[1], item[2])
self.__queue.task_done()
def __init__(self, *args, **kwargs):
super(DataManagerBase, self).__init__(*args, **kwargs)
# We want a dumb logger and dont really care about the output
self.logger = logging.getLogger()
self.logger.setLevel(logging.CRITICAL)
self.dbc, db_wrapper_manager = DbFactory.get_wrapper(mad_args)
if 'instance_id' in kwargs:
self.instance_id = kwargs['instance_id']
del kwargs['instance_id']
def __init__(self, device_mappings):
self.__mapping = {}
self.__playerstats = {}
self.__mapping_mutex = Lock()
self.__device_mappings = device_mappings
self.__injected = {}
self.__application_args = args
self.__db_wrapper: DbWrapperBase = DbFactory.get_wrapper(
self.__application_args)
if device_mappings is not None:
for origin in device_mappings.keys():
self.__mapping[origin] = {}
self.__playerstats[origin] = PlayerStats(
origin, self.__application_args, self.__db_wrapper)
self.__playerstats[origin].open_player_stats()
def run_receiver(self, listen_ip, listen_port, mitm_mapper, args_passed,
auths_passed):
global application_args, auths
application_args = args_passed
auths = auths_passed
self.__listen_ip = listen_ip
self.__listen_port = listen_port
self.__mitm_mapper: MitmMapper = mitm_mapper
self.app = Flask("MITMReceiver")
self.add_endpoint(endpoint='/',
endpoint_name='receive_protos',
handler=self.proto_endpoint,
methods_passed=['POST'])
self.add_endpoint(endpoint='/get_latest_mitm/',
endpoint_name='get_latest_mitm/',
handler=self.get_latest,
methods_passed=['GET'])
self.add_endpoint(endpoint='/get_addresses/',
endpoint_name='get_addresses/',
handler=self.get_addresses,
methods_passed=['GET'])
self._data_queue: JoinableQueue = JoinableQueue()
self._db_wrapper = DbFactory.get_wrapper(args_passed)
self.worker_threads = []
for i in range(application_args.mitmreceiver_data_workers):
data_processor: MitmDataProcessor = MitmDataProcessor()
t = Process(name='MITMReceiver-%s' % str(i),
target=data_processor.received_data_worker,
args=(data_processor, self._data_queue,
application_args, self.__mitm_mapper))
t.start()
self.worker_threads.append(t)
httpsrv = WSGIServer((self.__listen_ip, int(self.__listen_port)),
self.app.wsgi_app,
log=LogLevelChanger)
try:
httpsrv.serve_forever()
except KeyboardInterrupt as e:
logger.info("Stopping MITMReceiver")
httpsrv.close()
try:
pkg_resources.require(deps)
except pkg_resources.VersionConflict as version_error:
logger.error(
"Some dependencies aren't met. Required: {} (Installed: {})",
version_error.req, version_error.dist)
sys.exit(1)
if __name__ == "__main__":
check_dependencies()
# TODO: globally destroy all threads upon sys.exit() for example
install_thread_excepthook()
db_wrapper, db_wrapper_manager = DbFactory.get_wrapper(args)
db_wrapper.check_and_create_spawn_tables()
db_wrapper.create_quest_database_if_not_exists()
db_wrapper.create_status_database_if_not_exists()
db_wrapper.create_usage_database_if_not_exists()
db_wrapper.create_statistics_databases_if_not_exists()
version = MADVersion(args, db_wrapper)
version.get_version()
# create folders
create_folder(args.raidscreen_path)
create_folder(args.file_path)
create_folder(args.upload_path)
if args.only_ocr:
logger.error(
deviceUpdater, jobstatus)
if __name__ == "__main__":
logger.info('Starting MAD config mode')
filename = os.path.join('configs', 'config.ini')
if not os.path.exists(filename):
logger.error(
'config.ini file not found. Check configs folder and copy example config'
)
sys.exit(1)
create_folder(args.file_path)
create_folder(args.upload_path)
db_wrapper, db_pool_manager = DbFactory.get_wrapper(args)
instance_id = db_wrapper.get_instance_id()
data_manager = utils.data_manager.DataManager(db_wrapper, instance_id)
db_wrapper, db_pool_manager = DbFactory.get_wrapper(args)
version = MADVersion(args, data_manager)
version.get_version()
MappingManagerManager.register('MappingManager', MappingManager)
mapping_manager_manager = MappingManagerManager()
mapping_manager_manager.start()
mapping_manager_stop_event = mapping_manager_manager.Event()
mapping_manager: MappingManager = mapping_manager_manager.MappingManager(
db_wrapper, args, data_manager, True)
ws_server = WebsocketServer(args,
os.makedirs(folder)
def load_mappings(db_wrapper):
mapping_parser = MappingParser(db_wrapper, args, configmode=False)
device_mappings = mapping_parser.get_devicemappings()
routemanagers = mapping_parser.get_routemanagers()
auths = mapping_parser.get_auths()
return (device_mappings, routemanagers, auths)
if __name__ == "__main__":
# TODO: globally destroy all threads upon sys.exit() for example
install_thread_excepthook()
db_wrapper: DbWrapperBase = DbFactory.get_wrapper(args)
db_wrapper.create_hash_database_if_not_exists()
db_wrapper.check_and_create_spawn_tables()
db_wrapper.create_quest_database_if_not_exists()
db_wrapper.create_status_database_if_not_exists()
db_wrapper.create_usage_database_if_not_exists()
db_wrapper.create_statistics_databases_if_not_exists()
version = MADVersion(args, db_wrapper)
version.get_version()
if args.clean_hash_database:
logger.info('Cleanup Hash Database and www_hash folder')
db_wrapper.delete_hash_table('999', '')
for file in glob.glob("ocr/www_hash/*.jpg"):
os.remove(file)
sys.exit(0)