def __init__(self, message_queue, flag_shutdown, process_id=None): """Creates an instance of EventSenderAmqp Args: message_queue(queue.Queue): handler of a queue for sending events flag_shutdown(threading.Event): flag for handling shutdown process_id(str): process identifier (process name) """ super(EventSenderAmqp, self).__init__() self.__queue = message_queue self.__flag_shutdown = flag_shutdown self.__process_id = process_id # get configuration self.__config_mq = SystemConfigReader().get_all_values_from_section( 'MessageQueueing') self.__config_host = self.__config_mq.get("host", "127.0.0.1") self.__config_port = self.__config_mq.get("port", "5672") self.__config_username = self.__config_mq.get("username", "guest") self.__config_password = self.__config_mq.get("password", "guest") self.__config_exchange = self.__config_mq.get("exchange", "datagerry.eventbus") self.__config_retries = int( self.__config_mq.get("connection_attempts", "5")) self.__config_retrydelay = int(self.__config_mq.get( "retry_delay", "6")) self.__config_tls = False if self.__config_mq.get("use_tls", "False") in ("true", "True", "1", "yes"): self.__config_tls = True # define variables self.__connection = None self.__channel = None
def _check_database(): """ Checks whether the specified connection of the configuration is reachable. Returns: Datebase if response """ from cmdb.database.errors.connection_errors import ServerTimeoutError ssc = SystemConfigReader() LOGGER.info(f'Checking database connection with {ssc.config_name} data') database_options = ssc.get_all_values_from_section('Database') dbm = DatabaseManagerMongo(**database_options) try: connection_test = dbm.connector.is_connected() except ServerTimeoutError: connection_test = False LOGGER.debug(f'Database status is {connection_test}') if connection_test is True: return dbm retries = 0 while retries < 3: retries += 1 LOGGER.warning( f'Retry {retries}: Checking database connection with {SystemConfigReader.DEFAULT_CONFIG_NAME} data' ) connection_test = dbm.connector.is_connected() if connection_test: return dbm return None
def __init__(self, key_directory=None): self.scr = SystemConfigReader() ssc = SystemConfigReader() database_options = ssc.get_all_values_from_section('Database') self.__dbm = DatabaseManagerMongo(**database_options) self.ssw = SystemSettingsWriter(self.__dbm) self.key_directory = key_directory or SystemConfigReader.DEFAULT_CONFIG_LOCATION + "/keys"
def reload_config_reader(request_user: User): ssc = SystemConfigReader() ssc.setup() LOGGER.warning('Reload config file!') status = ssc.status() if status: current_app.cache.clear() return make_response(status)
def __init__(self): scr = SystemConfigReader() self.database_manager = DatabaseManagerMongo( **scr.get_all_values_from_section('Database')) self.object_manager = CmdbObjectManager( database_manager=self.database_manager) self.log_manager = ExportdLogManager( database_manager=self.database_manager) super().__init__(self.database_manager)
def __init__(self, key_directory=None): """ Args: key_directory: key based directory """ self.scr = SystemConfigReader() self.__dbm = DatabaseManagerMongo( **self.scr.get_all_values_from_section('Database')) self.ssr = SystemSettingsReader(self.__dbm) self.rsa_public = self.get_public_key() self.rsa_private = self.get_private_key()
def __init__(self): self.status = UpdateRoutine.UpateStatus.NOT # check if settings are loaded from cmdb.utils.system_config import SystemConfigReader self.setup_system_config_reader = SystemConfigReader() system_config_reader_status = self.setup_system_config_reader.status() if system_config_reader_status is not True: self.status = UpdateRoutine.UpateStatus.ERROR raise RuntimeError( f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \ system config reader status: {system_config_reader_status}')
def __init__(self, dbm: DatabaseManagerMongo): self.status = SetupRoutine.SetupStatus.NOT # check if settings are loaded self.setup_system_config_reader = SystemConfigReader() self.setup_database_manager = dbm system_config_reader_status = self.setup_system_config_reader.status() if system_config_reader_status is not True: self.status = SetupRoutine.SetupStatus.ERROR raise RuntimeError( f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \ system config reader status: {system_config_reader_status}' )
def __init__(self, job: ExportdJob): self.job = job self.exportvars = self.__get_exportvars() self.destinations = self.__get__destinations() scr = SystemConfigReader() database_manager = DatabaseManagerMongo( **scr.get_all_values_from_section('Database')) self.__object_manager = CmdbObjectManager( database_manager=database_manager) self.log_manager = ExportdLogManager(database_manager=database_manager) self.sources = self.__get_sources() super(ExportdManagerBase, self).__init__(database_manager)
def __init__(self, file, file_type, config: ObjectImporterConfig = None, parser: BaseObjectParser = None, object_manager: CmdbObjectManager = None, request_user: UserModel = None): """ Basic importer super class for object imports Normally should be started by start_import Args: file: File instance, name, content or loaded path to file file_type: file type - used with content-type config: importer configuration parser: the parser instance based on content-type object_manager: a instance of the object managers request_user: the instance of the started user """ self.parser = parser if object_manager: self.object_manager = object_manager else: from cmdb.utils.system_config import SystemConfigReader from cmdb.database.managers import DatabaseManagerMongo object_manager = CmdbObjectManager( database_manager=DatabaseManagerMongo(**SystemConfigReader( ).get_all_values_from_section('Database'))) self.object_manager = object_manager self.request_user = request_user super(ObjectImporter, self).__init__(file=file, file_type=file_type, config=config)
def __init__(self, database_manager: DatabaseManagerMongo = None): self.key_holder = KeyHolder() self.header = {'alg': 'RS512'} self.database_manager = database_manager or DatabaseManagerMongo( **SystemConfigReader().get_all_values_from_section('Database')) self.auth_module = AuthModule( SystemSettingsReader(self.database_manager))
def authenticate(self, user_name: str, password: str, **kwargs) -> UserModel: __dbm = DatabaseManagerMongo( **SystemConfigReader().get_all_values_from_section('Database') ) __user_manager = UserManager(__dbm) try: ldap_connection_status = self.connect() LOGGER.debug(f'[LdapAuthenticationProvider] Connection status: {ldap_connection_status}') except Exception as e: LOGGER.error(f'[LdapAuthenticationProvider] Failed to connect to LDAP server - error: {e}') raise AuthenticationError(LdapAuthenticationProvider.get_name(), e) ldap_search_filter = self.config.search['searchfilter'].replace("%username%", user_name) LOGGER.debug(f'[LdapAuthenticationProvider] Search Filter: {ldap_search_filter}') search_result = self.__ldap_connection.search(self.config.search['basedn'], ldap_search_filter) LOGGER.debug(f'[LdapAuthenticationProvider] Search result: {search_result}') if not search_result or len(self.__ldap_connection.entries) == 0: raise AuthenticationError(LdapAuthenticationProvider.get_name(), 'No matching entry') for entry in self.__ldap_connection.entries: LOGGER.debug(f'[LdapAuthenticationProvider] Entry: {entry}') entry_dn = entry.entry_dn try: entry_connection_result = LdapAuthenticationProvider.Connection(self.__ldap_server, entry_dn, password, auto_bind=True) LOGGER.debug(f'[LdapAuthenticationProvider] UserModel connection result: {entry_connection_result}') except Exception as e: LOGGER.error(f'[LdapAuthenticationProvider] UserModel auth result: {e}') raise AuthenticationError(LdapAuthenticationProvider.get_name(), e) # Check if user exists try: user_instance: UserModel = __user_manager.get_by({'user_name': user_name}) except ManagerGetError as umge: LOGGER.warning(f'[LdapAuthenticationProvider] UserModel exists on LDAP but not in database: {umge}') LOGGER.debug(f'[LdapAuthenticationProvider] Try creating user: {user_name}') try: new_user_data = dict() new_user_data['user_name'] = user_name new_user_data['active'] = True new_user_data['group_id'] = self.config.default_group new_user_data['registration_time'] = datetime.now() new_user_data['authenticator'] = LdapAuthenticationProvider.get_name() except Exception as e: LOGGER.debug(f'[LdapAuthenticationProvider] {e}') raise AuthenticationError(LdapAuthenticationProvider.get_name(), e) LOGGER.debug(f'[LdapAuthenticationProvider] New user was init') try: user_id = __user_manager.insert(new_user_data) except ManagerInsertError as umie: LOGGER.debug(f'[LdapAuthenticationProvider] {umie}') raise AuthenticationError(LdapAuthenticationProvider.get_name(), umie) try: user_instance: UserModel = __user_manager.get(public_id=user_id) except ManagerGetError as umge: LOGGER.debug(f'[LdapAuthenticationProvider] {umge}') raise AuthenticationError(LdapAuthenticationProvider.get_name(), umge) return user_instance
def __init__(self, event, state=False): super(ExportdThread, self).__init__() self.job = None self.job_id = event.get_param("id") self.type_id = event.get_param("type_id") self.user_id = event.get_param("user_id") self.event = event self.is_active = state self.exception_handling = None scr = SystemConfigReader() database_options = scr.get_all_values_from_section('Database') self.__dbm = DatabaseManagerMongo(**database_options) self.log_manager = ExportdLogManager(database_manager=self.__dbm) self.exportd_job_manager = ExportdJobManagement( database_manager=self.__dbm) self.user_manager = UserManager(database_manager=self.__dbm)
def _init_config_reader(config_file): import os path, filename = os.path.split(config_file) if filename is not SystemConfigReader.DEFAULT_CONFIG_NAME or path is not SystemConfigReader.DEFAULT_CONFIG_LOCATION: SystemConfigReader.RUNNING_CONFIG_NAME = filename SystemConfigReader.RUNNING_CONFIG_LOCATION = path + '/' SystemConfigReader(SystemConfigReader.RUNNING_CONFIG_NAME, SystemConfigReader.RUNNING_CONFIG_LOCATION)
def __init__(self, receiver_callback, flag_shutdown, process_id=None, event_types=["#"]): """Creates an instance of EventReceiverAmqp Args: receiver_callback(func): callback function for processing received events flag_shutdown(threading.Event): flag for handling shutdown process_id(str): process identifier (process name) event_types(list): list of event types, that will be processed """ super(EventReceiverAmqp, self).__init__() self.__receiver_callback = receiver_callback self.__flag_shutdown = flag_shutdown self.__process_id = process_id self.__event_types = event_types # get configuration self.__config_mq = SystemConfigReader().get_all_values_from_section( 'MessageQueueing') self.__config_host = self.__config_mq.get("host", "127.0.0.1") self.__config_port = self.__config_mq.get("port", "5672") self.__config_username = self.__config_mq.get("username", "guest") self.__config_password = self.__config_mq.get("password", "guest") self.__config_exchange = self.__config_mq.get("exchange", "datagerry.eventbus") self.__config_retries = int( self.__config_mq.get("connection_attempts", "5")) self.__config_retrydelay = int(self.__config_mq.get( "retry_delay", "6")) self.__config_tls = False if self.__config_mq.get("use_tls", "False") in ("true", "True", "1", "yes"): self.__config_tls = True # define variables self.__connection = None self.__channel = None
def __init__(self, object_list: List[CmdbObject], request_user: UserModel, dt_render=False, ref_render=False, object_manager: CmdbObjectManager = None): self.object_list: List[CmdbObject] = object_list self.request_user = request_user self.dt_render = dt_render self.ref_render = ref_render from cmdb.utils.system_config import SystemConfigReader database_manager = DatabaseManagerMongo( **SystemConfigReader().get_all_values_from_section('Database') ) self.object_manager = object_manager or CmdbObjectManager(database_manager=database_manager) self.user_manager = UserManager(database_manager=database_manager)
def __init__(self, event: Event, state: bool = False): scr = SystemConfigReader() database_options = scr.get_all_values_from_section('Database') database = DatabaseManagerMongo(**database_options) super(ExportdThread, self).__init__() self.job = None self.job_id = int(event.get_param("id")) self.type_id = int( event.get_param("type_id")) if event.get_param("type_id") else None self.user_id = int(event.get_param("user_id")) self.event = event self.is_active = state self.exception_handling = None self.object_manager = CmdbObjectManager(database_manager=database) self.log_manager = ExportdLogManager(database_manager=database) self.exportd_job_manager = ExportdJobManagement( database_manager=database) self.user_manager = UserManager(database_manager=database)
def _run(self): # get queue for sending events event_queue = self._event_manager.get_send_queue() database_manager = DatabaseManagerMongo( **SystemConfigReader().get_all_values_from_section('Database')) # get WSGI app app = DispatcherMiddleware(app=create_app(), mounts={ '/docs': create_docs_server(), '/rest': create_rest_api(database_manager, event_queue) }) # get gunicorn options options = SystemConfigReader().get_all_values_from_section('WebServer') # start gunicorn as own process webserver = HTTPServer(app, options) self.__webserver_proc = multiprocessing.Process(target=webserver.run) self.__webserver_proc.start() self.__webserver_proc.join()
def authenticate(self, user_name: str, password: str, **kwargs) -> UserModel: __dbm = DatabaseManagerMongo( **SystemConfigReader().get_all_values_from_section('Database') ) __scm = SecurityManager(__dbm) __user_manager = UserManager(__dbm) LOGGER.info(f'[LocalAuthenticationProvider] Try login for user {user_name}') try: user: UserModel = __user_manager.get_by({'user_name': user_name}) except ManagerGetError as umge: raise AuthenticationError(LocalAuthenticationProvider.get_name(), umge.message) login_pass = __scm.generate_hmac(password) if login_pass == user.password: return user raise AuthenticationError(LocalAuthenticationProvider.get_name(), 'UserModel not exists')
def get_config_information(request_user: User): ssc = SystemConfigReader() config_dict = {'path': ssc.config_file, 'properties': []} ssc = SystemConfigReader() for section in ssc.get_sections(): section_values = [] for key, value in ssc.get_all_values_from_section(section).items(): section_values.append([key, value]) config_dict['properties'].append([section, section_values]) if len(config_dict) < 1: return make_response(config_dict, 204) return make_response(config_dict)
class KeyHolder: def __init__(self, key_directory=None): """ Args: key_directory: key based directory """ self.scr = SystemConfigReader() self.__dbm = DatabaseManagerMongo( **self.scr.get_all_values_from_section('Database')) self.ssr = SystemSettingsReader(self.__dbm) self.rsa_public = self.get_public_key() self.rsa_private = self.get_private_key() def get_public_key(self): return self.ssr.get_value('asymmetric_key', 'security')['public'] def get_private_key(self): return self.ssr.get_value('asymmetric_key', 'security')['private']
class SetupRoutine: class SetupStatus(Enum): NOT = 0 RUNNING = 1 ERROR = 2 FINISHED = 3 def __init__(self, dbm: DatabaseManagerMongo): self.status = SetupRoutine.SetupStatus.NOT # check if settings are loaded self.setup_system_config_reader = SystemConfigReader() self.setup_database_manager = dbm system_config_reader_status = self.setup_system_config_reader.status() if system_config_reader_status is not True: self.status = SetupRoutine.SetupStatus.ERROR raise RuntimeError( f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \ system config reader status: {system_config_reader_status}' ) def get_setup_status(self): return self.status def setup(self) -> SetupStatus: LOGGER.info('SETUP ROUTINE: STARTED...') self.status = SetupRoutine.SetupStatus.RUNNING # check database if not self.__check_database(): self.status = SetupRoutine.SetupStatus.ERROR raise RuntimeError( 'The database managers could not be initialized. Perhaps the database cannot be reached, \ or the database was already initialized.') if self.__is_database_empty(): # init database try: self.__init_database() except Exception as err: self.status = SetupRoutine.SetupStatus.ERROR raise RuntimeError( f'Something went wrong during the initialization of the database. \n Error: {err}' ) # generate keys LOGGER.info('SETUP ROUTINE: Generate rsa key pair') try: self.init_keys() except Exception as err: self.status = SetupRoutine.SetupStatus.ERROR raise RuntimeError( f'Something went wrong during the generation of the rsa keypair. \n Error: {err}' ) # create user management LOGGER.info('SETUP ROUTINE: UserModel management') try: self.__create_user_management() except Exception as err: self.status = SetupRoutine.SetupStatus.ERROR raise RuntimeError( f'Something went wrong during the generation of the user management. \n Error: {err}' ) # create version updater settings try: from cmdb.updater import UpdaterModule from cmdb.updater.updater_settings import UpdateSettings from cmdb.utils.system_reader import SystemSettingsReader from cmdb.utils.system_writer import SystemSettingsWriter system_setting_writer: SystemSettingsWriter = SystemSettingsWriter( self.setup_database_manager) updater_setting_instance = UpdateSettings( **UpdaterModule.get_last_version()) system_setting_writer.write( _id='updater', data=updater_setting_instance.__dict__) except Exception as err: self.status = SetupRoutine.SetupStatus.ERROR raise RuntimeError( f'Something went wrong during the generation of the updater module. \n Error: {err}' ) self.status = SetupRoutine.SetupStatus.FINISHED LOGGER.info('SETUP ROUTINE: FINISHED!') return self.status def init_keys(self): from cmdb.security.key.generator import KeyGenerator kg = KeyGenerator() LOGGER.info('KEY ROUTINE: Generate RSA keypair') kg.generate_rsa_keypair() LOGGER.info('KEY ROUTINE: Generate aes key') kg.generate_symmetric_aes_key() self.__check_database() from cmdb.user_management.managers.user_manager import UserManager, UserModel from cmdb.security.security import SecurityManager scm = SecurityManager(self.setup_database_manager) usm = UserManager(self.setup_database_manager) try: admin_user: UserModel = usm.get(1) LOGGER.warning('KEY ROUTINE: Admin user detected') LOGGER.info( f'KEY ROUTINE: Enter new password for user: {admin_user.user_name}' ) admin_pass = str(input('New admin password: '******'KEY ROUTINE: Password was updated for user: {admin_user.user_name}' ) except Exception as ex: LOGGER.info( f'KEY ROUTINE: Password was updated for user failed: {ex}') LOGGER.info('KEY ROUTINE: FINISHED') def __create_user_management(self): from cmdb.user_management.models.user import UserModel from cmdb.user_management.managers.user_manager import UserManager from cmdb.user_management.managers.group_manager import GroupManager from cmdb.user_management import __FIXED_GROUPS__ from cmdb.security.security import SecurityManager scm = SecurityManager(self.setup_database_manager) group_manager = GroupManager(self.setup_database_manager) user_manager = UserManager(self.setup_database_manager) for group in __FIXED_GROUPS__: group_manager.insert(group) # setting the initial user to admin/admin as default admin_name = 'admin' admin_pass = '******' import datetime admin_user = UserModel( public_id=1, user_name=admin_name, active=True, group_id=__FIXED_GROUPS__[0].get_public_id(), registration_time=datetime.datetime.now(), password=scm.generate_hmac(admin_pass), ) user_manager.insert(admin_user) return True def __check_database(self): LOGGER.info('SETUP ROUTINE: Checking database connection') from cmdb.database.errors.connection_errors import ServerTimeoutError try: connection_test = self.setup_database_manager.connector.is_connected( ) except ServerTimeoutError: connection_test = False LOGGER.info( f'SETUP ROUTINE: Database connection status {connection_test}') return connection_test def __is_database_empty(self) -> bool: return not self.setup_database_manager.connector.database.list_collection_names( ) def __init_database(self): database_name = self.setup_system_config_reader.get_value( 'database_name', 'Database') LOGGER.info(f'SETUP ROUTINE: initialize database {database_name}') # delete database try: self.setup_database_manager.drop_database(database_name) except DatabaseNotExists as err: LOGGER.error(err.message) # create new database self.setup_database_manager.create_database(database_name) # generate collections # framework collections from cmdb.framework import __COLLECTIONS__ as FRAMEWORK_CLASSES for collection in FRAMEWORK_CLASSES: self.setup_database_manager.create_collection( collection.COLLECTION) # set unique indexes self.setup_database_manager.create_indexes( collection.COLLECTION, collection.get_index_keys()) # user management collections from cmdb.user_management import __COLLECTIONS__ as USER_MANAGEMENT_COLLECTION for collection in USER_MANAGEMENT_COLLECTION: self.setup_database_manager.create_collection( collection.COLLECTION) # set unique indexes self.setup_database_manager.create_indexes( collection.COLLECTION, collection.get_index_keys()) # ExportdJob management collections from cmdb.exportd import __COLLECTIONS__ as JOB_MANAGEMENT_COLLECTION for collection in JOB_MANAGEMENT_COLLECTION: self.setup_database_manager.create_collection( collection.COLLECTION) # set unique indexes self.setup_database_manager.create_indexes( collection.COLLECTION, collection.get_index_keys()) LOGGER.info('SETUP ROUTINE: initialize finished')
# You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. import logging import cmdb from cmdb.interface.cmdb_app import BaseCmdbApp from cmdb.interface.config import app_config from cmdb.interface.docs.doc_routes import doc_pages from cmdb.utils.error import CMDBError LOGGER = logging.getLogger(__name__) from cmdb.utils.system_config import SystemConfigReader system_config_reader = SystemConfigReader() def create_docs_server(event_queue): # Create managers from cmdb.data_storage.database_manager import DatabaseManagerMongo app_database = DatabaseManagerMongo( **system_config_reader.get_all_values_from_section('Database') ) app = BaseCmdbApp(__name__, app_database) if cmdb.__MODE__ == 'DEBUG': config = app_config['development'] config.APPLICATION_ROOT = '/docs/' app.config.from_object(config)
def test_config_less_instance(self): from cmdb.utils.system_config import SystemConfigReader system_config_reader = SystemConfigReader() assert system_config_reader.config_file_less is True
def config_reader(pytestconfig: Config) -> SystemConfigReader: config_path = pytestconfig.getoption('config-path', '../../etc/cmdb_test.conf') return SystemConfigReader.from_full_path(config_path)
from flask import abort, Response from cmdb.data_storage.database_manager import DatabaseManagerMongo from cmdb.framework.cmdb_errors import ObjectNotFoundError, TypeNotFoundError from cmdb.file_export.export_types import ExportType from cmdb.framework.cmdb_object_manager import CmdbObjectManager try: from cmdb.utils.error import CMDBError except ImportError: CMDBError = Exception from cmdb.utils.system_config import SystemConfigReader object_manager = CmdbObjectManager(database_manager=DatabaseManagerMongo( **SystemConfigReader().get_all_values_from_section('Database'))) class FileExporter: @staticmethod def get_filetypes(): filetypes = [ "CsvExportType", "JsonExportType", "XlsxExportType", "XmlExportType" ] return filetypes def __init__(self, object_type, export_type: ExportType, public_id: str, **kwargs): """init of FileExporter
class EventSenderAmqp(threading.Thread): """EventSender part of the EventManagerAmqp This part of the EventManagerAmqp is responsible for sending events to the message broker using the protocol AMQP. """ def __init__(self, message_queue, flag_shutdown, process_id=None): """Creates an instance of EventSenderAmqp Args: message_queue(queue.Queue): handler of a queue for sending events flag_shutdown(threading.Event): flag for handling shutdown process_id(str): process identifier (process name) """ super(EventSenderAmqp, self).__init__() self.__queue = message_queue self.__flag_shutdown = flag_shutdown self.__process_id = process_id # get configuration self.__config_mq = SystemConfigReader().get_all_values_from_section( 'MessageQueueing') self.__config_host = self.__config_mq.get("host", "127.0.0.1") self.__config_port = self.__config_mq.get("port", "5672") self.__config_username = self.__config_mq.get("username", "guest") self.__config_password = self.__config_mq.get("password", "guest") self.__config_exchange = self.__config_mq.get("exchange", "datagerry.eventbus") self.__config_retries = int( self.__config_mq.get("connection_attempts", "5")) self.__config_retrydelay = int(self.__config_mq.get( "retry_delay", "6")) self.__config_tls = False if self.__config_mq.get("use_tls", "False") in ("true", "True", "1", "yes"): self.__config_tls = True # define variables self.__connection = None self.__channel = None def __init_connection(self): """create a connection to message broker""" try: credentials = pika.credentials.PlainCredentials( self.__config_username, self.__config_password) self.__connection = pika.BlockingConnection( pika.ConnectionParameters( host=self.__config_host, port=self.__config_port, connection_attempts=self.__config_retries, retry_delay=self.__config_retrydelay, credentials=credentials, ssl=self.__config_tls)) self.__channel = self.__connection.channel() self.__channel.exchange_declare(exchange=self.__config_exchange, exchange_type="topic") except pika.exceptions.AMQPConnectionError: LOGGER.error("{}: EventSenderAmqp connection error".format( self.__process_id)) self.__flag_shutdown.set() def run(self): """run the event sender""" # init connection to broker self.__init_connection() # check queue for new events while not self.__flag_shutdown.is_set(): try: try: event = self.__queue.get(block=True, timeout=2) event_type = event.get_type() event_serialized = event.json_repr() self.__channel.basic_publish( exchange=self.__config_exchange, routing_key=event_type, body=event_serialized) except queue.Empty: self.__connection.process_data_events() # handle AMQP connection errors except pika.exceptions.AMQPConnectionError: LOGGER.warning( "connection to broker lost, try to reconnect...") self.__init_connection()
class EventReceiverAmqp(threading.Thread): """EventReceiver part of the EventManagerAmqp This part of the EventManagerAmqp is responsible for receiving events from the message broker using the protocol AMQP. Only configured event types were processed (e.g. cmdb.core.objects.added) """ def __init__(self, receiver_callback, flag_shutdown, process_id=None, event_types=["#"]): """Creates an instance of EventReceiverAmqp Args: receiver_callback(func): callback function for processing received events flag_shutdown(threading.Event): flag for handling shutdown process_id(str): process identifier (process name) event_types(list): list of event types, that will be processed """ super(EventReceiverAmqp, self).__init__() self.__receiver_callback = receiver_callback self.__flag_shutdown = flag_shutdown self.__process_id = process_id self.__event_types = event_types # get configuration self.__config_mq = SystemConfigReader().get_all_values_from_section( 'MessageQueueing') self.__config_host = self.__config_mq.get("host", "127.0.0.1") self.__config_port = self.__config_mq.get("port", "5672") self.__config_username = self.__config_mq.get("username", "guest") self.__config_password = self.__config_mq.get("password", "guest") self.__config_exchange = self.__config_mq.get("exchange", "datagerry.eventbus") self.__config_retries = int( self.__config_mq.get("connection_attempts", "5")) self.__config_retrydelay = int(self.__config_mq.get( "retry_delay", "6")) self.__config_tls = False if self.__config_mq.get("use_tls", "False") in ("true", "True", "1", "yes"): self.__config_tls = True # define variables self.__connection = None self.__channel = None def __process_event_cb(self, ch, method, properties, body): """event processing this callback function is executed, if an event was reveived. Args: ch: AMQP channel method: AMQP method properties: AMQP properties body: AMQP message body """ event = Event.create_event(body) # allow None values, if event receiving should be ignored if self.__receiver_callback: self.__receiver_callback(event) def __check_shutdown_flag(self): """check, if the shutdown flag was set""" # reinstall shutdown check self.__connection.add_timeout(2, self.__check_shutdown_flag) # check shutdown flag if self.__flag_shutdown.is_set(): self.__channel.stop_consuming() self.__connection.close() def __init_connection(self): """create a connection to message broker""" try: # init connection to broker credentials = pika.credentials.PlainCredentials( self.__config_username, self.__config_password) self.__connection = pika.BlockingConnection( pika.ConnectionParameters( host=self.__config_host, port=self.__config_port, connection_attempts=self.__config_retries, retry_delay=self.__config_retrydelay, credentials=credentials, ssl=self.__config_tls)) self.__channel = self.__connection.channel() self.__channel.exchange_declare(exchange=self.__config_exchange, exchange_type="topic") queue_name = "" if self.__process_id: queue_name = "{}.{}".format(self.__config_exchange, self.__process_id) queue_declare_result = self.__channel.queue_declare( queue=queue_name, exclusive=True) queue_handler = queue_declare_result.method.queue for event_type in self.__event_types: self.__channel.queue_bind(exchange=self.__config_exchange, queue=queue_handler, routing_key=event_type) # register callback function for event handling self.__channel.basic_consume(self.__process_event_cb, queue=queue_handler, no_ack=True) except AMQPConnectionError: LOGGER.error("{}: EventReceiverAmqp connection error".format( self.__process_id)) self.__flag_shutdown.set() def run(self): """run the event receiver""" # init connection to broker self.__init_connection() while not self.__flag_shutdown.is_set(): try: # start handling events self.__connection.add_timeout(2, self.__check_shutdown_flag) self.__channel.start_consuming() # handle AMQP connection errors except AMQPConnectionError: LOGGER.warning( "connection to broker lost, try to reconnect...") self.__init_connection()
def main(args): """ Default application start function Args: args: start-options """ LOGGER.info("DATAGERRY starting...") if args.debug: _activate_debug() _init_config_reader(args.config_file) from cmdb.data_storage.database_connection import DatabaseConnectionError # create / check connection database manager dbm = None try: dbm = _check_database() if not dbm: raise DatabaseConnectionError('') LOGGER.info("Database connection established.") except CMDBError as conn_error: LOGGER.critical(conn_error.message) exit(1) # check db-settings and run update if needed if args.start: from cmdb.__check__ import CheckRoutine check_routine = CheckRoutine(dbm) # check db-settings try: check_status = check_routine.checker() except Exception as err: LOGGER.error(err) check_status = check_routine.get_check_status() LOGGER.error( f'The check did not go through as expected. Please run an update. \n Error: {err}' ) if check_status == CheckRoutine.CheckStatus.HAS_UPDATES: # run update from cmdb.__update__ import UpdateRoutine update_routine = UpdateRoutine() try: update_status = update_routine.start_update() except RuntimeError as err: LOGGER.error(err) update_status = update_routine.get_updater_status() LOGGER.warning( f'The update did not go through as expected - Status {update_status}' ) if update_status == UpdateRoutine.UpateStatus.FINISHED: check_status = CheckRoutine.CheckStatus.FINISHED else: exit(1) if check_status == CheckRoutine.CheckStatus.FINISHED: # run setup if needed from cmdb.__setup__ import SetupRoutine setup_routine = SetupRoutine(dbm) try: setup_status = setup_routine.setup() except RuntimeError as err: LOGGER.error(err) setup_status = setup_routine.get_setup_status() LOGGER.warning( f'The setup did not go through as expected - Status {setup_status}' ) if setup_status == SetupRoutine.SetupStatus.FINISHED: pass else: exit(1) else: pass if args.keys: from cmdb.__setup__ import SetupRoutine setup_routine = SetupRoutine(dbm) setup_status = None try: setup_routine.init_keys() except RuntimeError as err: LOGGER.error(err) setup_status = setup_routine.get_setup_status() LOGGER.warning( f'The key generation did not go through as expected - Status {setup_status}' ) if setup_status == SetupRoutine.SetupStatus.FINISHED: exit(0) else: exit(1) if args.test_data: _activate_debug() from cmdb.utils.data_factory import DataFactory ssc = SystemConfigReader() database_options = ssc.get_all_values_from_section('Database') dbm = DatabaseManagerMongo(**database_options) db_name = dbm.get_database_name() LOGGER.warning(f'Inserting test-data into: {db_name}') try: factory = DataFactory(database_manager=dbm) ack = factory.insert_data() LOGGER.warning("Test-data was successfully added".format( dbm.get_database_name())) if len(ack) > 0: LOGGER.critical( "Error while inserting test-data: {} - dropping database". format(ack)) dbm.drop(db_name) # cleanup database except (Exception, CMDBError) as e: import traceback traceback.print_tb(e.__traceback__) dbm.drop(db_name) # cleanup database exit(1) if args.start: _start_app() sleep(0.2) # prevent logger output LOGGER.info("DATAGERRY successfully started")
class UpdateRoutine: class UpateStatus(Enum): NOT = 0 RUNNING = 1 ERROR = 2 FINISHED = 3 def __init__(self): self.status = UpdateRoutine.UpateStatus.NOT # check if settings are loaded from cmdb.utils.system_config import SystemConfigReader self.setup_system_config_reader = SystemConfigReader() system_config_reader_status = self.setup_system_config_reader.status() if system_config_reader_status is not True: self.status = UpdateRoutine.UpateStatus.ERROR raise RuntimeError( f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \ system config reader status: {system_config_reader_status}') def get_updater_status(self): return self.status def __check_database(self): LOGGER.info('SETUP ROUTINE: Checking database connection') from cmdb.data_storage.database_manager import DatabaseManagerMongo from cmdb.data_storage.database_connection import ServerTimeoutError try: self.setup_database_manager = DatabaseManagerMongo( **self.setup_system_config_reader.get_all_values_from_section('Database') ) connection_test = self.setup_database_manager.connector.is_connected() except ServerTimeoutError: connection_test = False LOGGER.info(f'SETUP ROUTINE: Database connection status {connection_test}') return connection_test def __is_database_empty(self) -> bool: return not self.setup_database_manager.connector.database.list_collection_names() def start_update(self): LOGGER.info('UPDATE ROUTINE: Update database collection') self.status = UpdateRoutine.UpateStatus.RUNNING # check database if not self.__check_database(): self.status = UpdateRoutine.UpateStatus.ERROR raise RuntimeError( 'The database manager could not be initialized. Perhaps the database cannot be reached, \ or the database was already initialized.' ) if not self.__is_database_empty(): self.update_database_collection() self.update_db_version() else: LOGGER.info('UPDATE ROUTINE: The update is faulty because no collection was detected.') LOGGER.info('UPDATE ROUTINE: Update database collection finished.') self.status = UpdateRoutine.UpateStatus.FINISHED LOGGER.info('UPDATE ROUTINE: FINISHED!') return self.status def update_database_collection(self): from cmdb.framework import __COLLECTIONS__ as FRAMEWORK_CLASSES from cmdb.user_management import __COLLECTIONS__ as USER_MANAGEMENT_COLLECTION from cmdb.exportd import __COLLECTIONS__ as JOB_MANAGEMENT_COLLECTION # update database collections try: detected_database = self.setup_database_manager.connector.database # update collections # framework collections for collection in FRAMEWORK_CLASSES: try: detected_database.validate_collection(collection.COLLECTION)['valid'] except: self.setup_database_manager.create_collection(collection.COLLECTION) # set unique indexes self.setup_database_manager.create_indexes(collection.COLLECTION, collection.get_index_keys()) LOGGER.info(f'UPDATE ROUTINE: Database collection {collection.COLLECTION} was created.') # user management collections for collection in USER_MANAGEMENT_COLLECTION: try: detected_database.validate_collection(collection.COLLECTION)['valid'] except: self.setup_database_manager.create_collection(collection.COLLECTION) # set unique indexes self.setup_database_manager.create_indexes(collection.COLLECTION, collection.get_index_keys()) LOGGER.info(f'UPDATE ROUTINE: Database collection {collection.COLLECTION} was created.') # exportdJob management collections for collection in JOB_MANAGEMENT_COLLECTION: try: detected_database.validate_collection(collection.COLLECTION)['valid'] except: self.setup_database_manager.create_collection(collection.COLLECTION) # set unique indexes self.setup_database_manager.create_indexes(collection.COLLECTION, collection.get_index_keys()) LOGGER.info(f'UPDATE ROUTINE: Database collection {collection.COLLECTION} was created.') except Exception as ex: LOGGER.info(f'UPDATE ROUTINE: Database collection validation failed: {ex}') def update_db_version(self): # update version updater settings try: updater_settings_values = UpdaterModule.__DEFAULT_SETTINGS__ ssr = SystemSettingsReader(self.setup_database_manager) try: updater_settings_values = ssr.get_all_values_from_section('updater') updater_setting_instance = UpdateSettings(**updater_settings_values) except Exception: # create updater section if not exist system_setting_writer: SystemSettingsWriter = SystemSettingsWriter(self.setup_database_manager) updater_setting_instance = UpdateSettings(**updater_settings_values) system_setting_writer.write(_id='updater', data=updater_setting_instance.__dict__) # start running update files updater_setting_instance.run_updates(updater_settings_values.get('version'), ssr) except Exception as err: self.status = UpdateRoutine.UpateStatus.ERROR raise RuntimeError( f'Something went wrong during the generation of the updater module. \n Error: {err}' )