class Service(ServiceBase):
""" Siterummage Big Broker microservice class """
## Title text logged during initialisation.
title_text = 'Site Rummagge Big Broker Microservice'
## Copyright text logged on initialisation etc.
copyright_text = 'Copyright 2021 Site Rummage'
## License text logged on initialisation etc.
license_text = 'All Rights Reserved. Proprietary and confidential'
def __init__(self, new_instance):
super().__init__()
self._quart = new_instance
## Instance of the logging wrapper class
self._logger = Logger()
## _is_initialised is inherited from parent class ServiceThread
self._is_initialised = False
self._configuration = None
self._api_schedule = None
self._api_node_management = None
self._scrape_node_list = ScrapeNodeList()
def _initialise(self) -> bool:
self._logger.write_to_console = True
self._logger.initialise()
self._logger.log(LogType.Info,
f'{self.title_text} {VERSION} (Core Version {CORE_VERSION})')
self._logger.log(LogType.Info, self.copyright_text)
self._logger.log(LogType.Info, self.license_text)
config_mgr = ConfigurationManager()
config_file = os.getenv('SITERUMMAGE_BIGBROKER_CONFIG')
self._logger.log(LogType.Info, f'Configuration file: {config_file}')
self._configuration = config_mgr.parse_config_file(config_file)
if not self._configuration:
self._logger.log(LogType.Error, config_mgr.last_error_msg)
return False
self._logger.log(LogType.Info, '+=== Configuration Settings ===+')
self._logger.log(LogType.Info, '+==============================+')
page_store_cfg = self._configuration.page_store_api
self._logger.log(LogType.Info, '+== Page Store Api :->')
self._logger.log(LogType.Info, f'+= host : {page_store_cfg.host}')
self._logger.log(LogType.Info, f'+= port : {page_store_cfg.port}')
processing_queue_cfg = self._configuration.processing_queue_api
self._logger.log(LogType.Info, '+== Page Store Api :->')
self._logger.log(LogType.Info, f'+= host : {processing_queue_cfg.host}')
self._logger.log(LogType.Info, f'+= port : {processing_queue_cfg.port}')
self._logger.log(LogType.Info, '+==============================+')
self._api_schedule = ApiSchedule(self._quart, self._configuration)
self._api_node_management = ApiNodeManagement(self._quart,
self._configuration,
self._scrape_node_list)
self._is_initialised = True
return True
async def _main_loop(self):
# if not self._master_thread_class.initialise():
# return False
pass
def _shutdown(self):
self._logger.log(LogType.Info, 'Shutting down...')
class Service(ServiceBase):
""" Siterummage Page Store microservice class """
## Title text logged during initialisation.
title_text = 'Site Rummagge Page Store Microservice'
## Copyright text logged on initialisation etc.
copyright_text = 'Copyright 2021 Site Rummage'
## License text logged on initialisation etc.
license_text = 'All Rights Reserved. Proprietary and confidential'
def __init__(self, new_instance):
super().__init__()
self._quart = new_instance
## Instance of the logging wrapper class
self._logger = Logger()
## _is_initialised is inherited from parent class ServiceThread
self._is_initialised = False
self._configuration = None
self._db_interface = None
self._api_health = ApiHealth(self._quart)
self._api_webpage = None
def _initialise(self) -> bool:
self._logger.write_to_console = True
self._logger.initialise()
self._logger.log(
LogType.Info,
f'{self.title_text} {VERSION} (Core Version {CORE_VERSION})')
self._logger.log(LogType.Info, self.copyright_text)
self._logger.log(LogType.Info, self.license_text)
config_mgr = ConfigurationManager()
config_file = os.getenv('SITERUMMAGE_PAGESTORE_CONFIG')
self._configuration = config_mgr.parse_config_file(config_file)
if not self._configuration:
self._logger.log(LogType.Error, config_mgr.last_error_msg)
return False
self._logger.log(LogType.Info, '+=== Configuration Settings ===+')
self._logger.log(LogType.Info, '+==============================+')
db_config = self._configuration.db_settings
self._logger.log(LogType.Info, '+== Database Settings :->')
self._logger.log(LogType.Info, f'+= database : {db_config.database}')
self._logger.log(LogType.Info, f'+= host : {db_config.host}')
self._logger.log(LogType.Info, f'+= username : {db_config.username}')
self._logger.log(LogType.Info, f'+= port : {db_config.port}')
self._logger.log(LogType.Info, f'+= pool_name : {db_config.pool_name}')
self._logger.log(LogType.Info, f'+= pool_size : {db_config.pool_size}')
self._logger.log(LogType.Info, '+==============================+')
self._db_interface = DatabaseInterface(self._logger,
self._configuration)
if not self._db_interface.database_connection_valid():
return False
self._api_webpage = ApiWebpage(self._quart, self._db_interface,
self._configuration)
self._is_initialised = True
return True
async def _main_loop(self):
# if not self._master_thread_class.initialise():
# return False
pass
def _shutdown(self):
self._logger.log(LogType.Info, 'Shutting down...')
class Service(ServiceBase):
""" Siterummage Processing Queue microservice class """
#pylint: disable=too-many-instance-attributes
## Title text logged during initialisation.
title_text = 'Site Rummagge Processing Queue Microservice'
## Copyright text logged on initialisation etc.
copyright_text = 'Copyright 2021 Site Rummage'
## License text logged on initialisation etc.
license_text = 'All Rights Reserved. Proprietary and confidential'
def __init__(self, new_instance):
super().__init__()
self._quart = new_instance
## Instance of the logging wrapper class
self._logger = Logger()
## _is_initialised is inherited from parent class ServiceThread
self._is_initialised = False
self._configuration = None
self._db_interface = None
self._api_queue = None
self._queue_cache = None
self._processing_queue = None
def _initialise(self) -> bool:
self._logger.write_to_console = True
self._logger.initialise()
self._logger.log(
LogType.Info,
f'{self.title_text} {VERSION} (Core Version {CORE_VERSION})')
self._logger.log(LogType.Info, self.copyright_text)
self._logger.log(LogType.Info, self.license_text)
config_mgr = ConfigurationManager()
config_file = os.getenv('SITERUMMAGE_PROCESSINGQUEUE_CONFIG')
self._configuration = config_mgr.parse_config_file(config_file)
if not self._configuration:
self._logger.log(LogType.Error, config_mgr.last_error_msg)
return False
self._logger.log(LogType.Info, '+=== Configuration Settings ===+')
self._logger.log(LogType.Info, '+==============================+')
db_config = self._configuration.db_settings
self._logger.log(LogType.Info, '+== Database Settings :->')
self._logger.log(LogType.Info,
f'+= Cache Size : {db_config.cache_size}')
self._logger.log(
LogType.Info,
f'+= DB Filename : {db_config.database_file}')
self._logger.log(
LogType.Info,
f'+= Fail On No Database : {db_config.fail_on_no_database}')
self._logger.log(LogType.Info, '+== Api Settings :->')
self._logger.log(LogType.Info, '+= Auth Key : ******')
self._logger.log(LogType.Info, '+==============================+')
self._db_interface = DbInterface(db_config.database_file)
if not self._db_interface.database_exists():
if self._configuration.db_settings.fail_on_no_database:
self._logger.log(LogType.Error,
"DB doesn't exist and fail on create is set")
return False
if not self._db_interface.build_database():
self._logger.log(LogType.Error,
self._db_interface.last_error_message)
return False
self._logger.log(LogType.Info, 'Database created successfully')
if not self._db_interface.open():
self._logger.log(LogType.Error,
self._db_interface.last_error_message)
return False
self._processing_queue = UrlsBeingProcessed()
self._queue_cache = QueueCache(self._db_interface, self._configuration,
self._logger, self._processing_queue)
self._api_queue = ApiQueue(self._quart, self._db_interface,
self._configuration, self._processing_queue,
self._queue_cache)
self._is_initialised = True
return True
async def _main_loop(self):
...
def _shutdown(self):
self._logger.log(LogType.Info, 'Shutting down...')
if self._db_interface.is_connected:
self._db_interface.close()
self._logger.log(LogType.Info, '|-> Database connection closed')