def __init__(self, message_queue, flag_shutdown, process_id=None):
"""Creates an instance of EventSenderAmqp
Args:
message_queue(queue.Queue): handler of a queue for sending events
flag_shutdown(threading.Event): flag for handling shutdown
process_id(str): process identifier (process name)
"""
super(EventSenderAmqp, self).__init__()
self.__queue = message_queue
self.__flag_shutdown = flag_shutdown
self.__process_id = process_id
# get configuration
self.__config_mq = SystemConfigReader().get_all_values_from_section(
'MessageQueueing')
self.__config_host = self.__config_mq.get("host", "127.0.0.1")
self.__config_port = self.__config_mq.get("port", "5672")
self.__config_username = self.__config_mq.get("username", "guest")
self.__config_password = self.__config_mq.get("password", "guest")
self.__config_exchange = self.__config_mq.get("exchange",
"datagerry.eventbus")
self.__config_retries = int(
self.__config_mq.get("connection_attempts", "5"))
self.__config_retrydelay = int(self.__config_mq.get(
"retry_delay", "6"))
self.__config_tls = False
if self.__config_mq.get("use_tls",
"False") in ("true", "True", "1", "yes"):
self.__config_tls = True
# define variables
self.__connection = None
self.__channel = None
def _check_database():
"""
Checks whether the specified connection of the configuration is reachable.
Returns: Datebase if response
"""
from cmdb.database.errors.connection_errors import ServerTimeoutError
ssc = SystemConfigReader()
LOGGER.info(f'Checking database connection with {ssc.config_name} data')
database_options = ssc.get_all_values_from_section('Database')
dbm = DatabaseManagerMongo(**database_options)
try:
connection_test = dbm.connector.is_connected()
except ServerTimeoutError:
connection_test = False
LOGGER.debug(f'Database status is {connection_test}')
if connection_test is True:
return dbm
retries = 0
while retries < 3:
retries += 1
LOGGER.warning(
f'Retry {retries}: Checking database connection with {SystemConfigReader.DEFAULT_CONFIG_NAME} data'
)
connection_test = dbm.connector.is_connected()
if connection_test:
return dbm
return None
def __init__(self, key_directory=None):
self.scr = SystemConfigReader()
ssc = SystemConfigReader()
database_options = ssc.get_all_values_from_section('Database')
self.__dbm = DatabaseManagerMongo(**database_options)
self.ssw = SystemSettingsWriter(self.__dbm)
self.key_directory = key_directory or SystemConfigReader.DEFAULT_CONFIG_LOCATION + "/keys"
def reload_config_reader(request_user: User):
ssc = SystemConfigReader()
ssc.setup()
LOGGER.warning('Reload config file!')
status = ssc.status()
if status:
current_app.cache.clear()
return make_response(status)
def __init__(self):
scr = SystemConfigReader()
self.database_manager = DatabaseManagerMongo(
**scr.get_all_values_from_section('Database'))
self.object_manager = CmdbObjectManager(
database_manager=self.database_manager)
self.log_manager = ExportdLogManager(
database_manager=self.database_manager)
super().__init__(self.database_manager)
def __init__(self, key_directory=None):
"""
Args:
key_directory: key based directory
"""
self.scr = SystemConfigReader()
self.__dbm = DatabaseManagerMongo(
**self.scr.get_all_values_from_section('Database'))
self.ssr = SystemSettingsReader(self.__dbm)
self.rsa_public = self.get_public_key()
self.rsa_private = self.get_private_key()
def __init__(self):
self.status = UpdateRoutine.UpateStatus.NOT
# check if settings are loaded
from cmdb.utils.system_config import SystemConfigReader
self.setup_system_config_reader = SystemConfigReader()
system_config_reader_status = self.setup_system_config_reader.status()
if system_config_reader_status is not True:
self.status = UpdateRoutine.UpateStatus.ERROR
raise RuntimeError(
f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \
system config reader status: {system_config_reader_status}')
def __init__(self, dbm: DatabaseManagerMongo):
self.status = SetupRoutine.SetupStatus.NOT
# check if settings are loaded
self.setup_system_config_reader = SystemConfigReader()
self.setup_database_manager = dbm
system_config_reader_status = self.setup_system_config_reader.status()
if system_config_reader_status is not True:
self.status = SetupRoutine.SetupStatus.ERROR
raise RuntimeError(
f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \
system config reader status: {system_config_reader_status}'
)
def __init__(self, job: ExportdJob):
self.job = job
self.exportvars = self.__get_exportvars()
self.destinations = self.__get__destinations()
scr = SystemConfigReader()
database_manager = DatabaseManagerMongo(
**scr.get_all_values_from_section('Database'))
self.__object_manager = CmdbObjectManager(
database_manager=database_manager)
self.log_manager = ExportdLogManager(database_manager=database_manager)
self.sources = self.__get_sources()
super(ExportdManagerBase, self).__init__(database_manager)
def __init__(self,
file,
file_type,
config: ObjectImporterConfig = None,
parser: BaseObjectParser = None,
object_manager: CmdbObjectManager = None,
request_user: UserModel = None):
"""
Basic importer super class for object imports
Normally should be started by start_import
Args:
file: File instance, name, content or loaded path to file
file_type: file type - used with content-type
config: importer configuration
parser: the parser instance based on content-type
object_manager: a instance of the object managers
request_user: the instance of the started user
"""
self.parser = parser
if object_manager:
self.object_manager = object_manager
else:
from cmdb.utils.system_config import SystemConfigReader
from cmdb.database.managers import DatabaseManagerMongo
object_manager = CmdbObjectManager(
database_manager=DatabaseManagerMongo(**SystemConfigReader(
).get_all_values_from_section('Database')))
self.object_manager = object_manager
self.request_user = request_user
super(ObjectImporter, self).__init__(file=file,
file_type=file_type,
config=config)
def __init__(self, database_manager: DatabaseManagerMongo = None):
self.key_holder = KeyHolder()
self.header = {'alg': 'RS512'}
self.database_manager = database_manager or DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database'))
self.auth_module = AuthModule(
SystemSettingsReader(self.database_manager))
def authenticate(self, user_name: str, password: str, **kwargs) -> UserModel:
__dbm = DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database')
)
__user_manager = UserManager(__dbm)
try:
ldap_connection_status = self.connect()
LOGGER.debug(f'[LdapAuthenticationProvider] Connection status: {ldap_connection_status}')
except Exception as e:
LOGGER.error(f'[LdapAuthenticationProvider] Failed to connect to LDAP server - error: {e}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), e)
ldap_search_filter = self.config.search['searchfilter'].replace("%username%", user_name)
LOGGER.debug(f'[LdapAuthenticationProvider] Search Filter: {ldap_search_filter}')
search_result = self.__ldap_connection.search(self.config.search['basedn'], ldap_search_filter)
LOGGER.debug(f'[LdapAuthenticationProvider] Search result: {search_result}')
if not search_result or len(self.__ldap_connection.entries) == 0:
raise AuthenticationError(LdapAuthenticationProvider.get_name(), 'No matching entry')
for entry in self.__ldap_connection.entries:
LOGGER.debug(f'[LdapAuthenticationProvider] Entry: {entry}')
entry_dn = entry.entry_dn
try:
entry_connection_result = LdapAuthenticationProvider.Connection(self.__ldap_server, entry_dn, password,
auto_bind=True)
LOGGER.debug(f'[LdapAuthenticationProvider] UserModel connection result: {entry_connection_result}')
except Exception as e:
LOGGER.error(f'[LdapAuthenticationProvider] UserModel auth result: {e}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), e)
# Check if user exists
try:
user_instance: UserModel = __user_manager.get_by({'user_name': user_name})
except ManagerGetError as umge:
LOGGER.warning(f'[LdapAuthenticationProvider] UserModel exists on LDAP but not in database: {umge}')
LOGGER.debug(f'[LdapAuthenticationProvider] Try creating user: {user_name}')
try:
new_user_data = dict()
new_user_data['user_name'] = user_name
new_user_data['active'] = True
new_user_data['group_id'] = self.config.default_group
new_user_data['registration_time'] = datetime.now()
new_user_data['authenticator'] = LdapAuthenticationProvider.get_name()
except Exception as e:
LOGGER.debug(f'[LdapAuthenticationProvider] {e}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), e)
LOGGER.debug(f'[LdapAuthenticationProvider] New user was init')
try:
user_id = __user_manager.insert(new_user_data)
except ManagerInsertError as umie:
LOGGER.debug(f'[LdapAuthenticationProvider] {umie}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), umie)
try:
user_instance: UserModel = __user_manager.get(public_id=user_id)
except ManagerGetError as umge:
LOGGER.debug(f'[LdapAuthenticationProvider] {umge}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), umge)
return user_instance
def __init__(self, event, state=False):
super(ExportdThread, self).__init__()
self.job = None
self.job_id = event.get_param("id")
self.type_id = event.get_param("type_id")
self.user_id = event.get_param("user_id")
self.event = event
self.is_active = state
self.exception_handling = None
scr = SystemConfigReader()
database_options = scr.get_all_values_from_section('Database')
self.__dbm = DatabaseManagerMongo(**database_options)
self.log_manager = ExportdLogManager(database_manager=self.__dbm)
self.exportd_job_manager = ExportdJobManagement(
database_manager=self.__dbm)
self.user_manager = UserManager(database_manager=self.__dbm)
def _init_config_reader(config_file):
import os
path, filename = os.path.split(config_file)
if filename is not SystemConfigReader.DEFAULT_CONFIG_NAME or path is not SystemConfigReader.DEFAULT_CONFIG_LOCATION:
SystemConfigReader.RUNNING_CONFIG_NAME = filename
SystemConfigReader.RUNNING_CONFIG_LOCATION = path + '/'
SystemConfigReader(SystemConfigReader.RUNNING_CONFIG_NAME,
SystemConfigReader.RUNNING_CONFIG_LOCATION)
def __init__(self,
receiver_callback,
flag_shutdown,
process_id=None,
event_types=["#"]):
"""Creates an instance of EventReceiverAmqp
Args:
receiver_callback(func): callback function for processing
received events
flag_shutdown(threading.Event): flag for handling shutdown
process_id(str): process identifier (process name)
event_types(list): list of event types, that will be processed
"""
super(EventReceiverAmqp, self).__init__()
self.__receiver_callback = receiver_callback
self.__flag_shutdown = flag_shutdown
self.__process_id = process_id
self.__event_types = event_types
# get configuration
self.__config_mq = SystemConfigReader().get_all_values_from_section(
'MessageQueueing')
self.__config_host = self.__config_mq.get("host", "127.0.0.1")
self.__config_port = self.__config_mq.get("port", "5672")
self.__config_username = self.__config_mq.get("username", "guest")
self.__config_password = self.__config_mq.get("password", "guest")
self.__config_exchange = self.__config_mq.get("exchange",
"datagerry.eventbus")
self.__config_retries = int(
self.__config_mq.get("connection_attempts", "5"))
self.__config_retrydelay = int(self.__config_mq.get(
"retry_delay", "6"))
self.__config_tls = False
if self.__config_mq.get("use_tls",
"False") in ("true", "True", "1", "yes"):
self.__config_tls = True
# define variables
self.__connection = None
self.__channel = None
def __init__(self, object_list: List[CmdbObject], request_user: UserModel, dt_render=False, ref_render=False,
object_manager: CmdbObjectManager = None):
self.object_list: List[CmdbObject] = object_list
self.request_user = request_user
self.dt_render = dt_render
self.ref_render = ref_render
from cmdb.utils.system_config import SystemConfigReader
database_manager = DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database')
)
self.object_manager = object_manager or CmdbObjectManager(database_manager=database_manager)
self.user_manager = UserManager(database_manager=database_manager)
def __init__(self, event: Event, state: bool = False):
scr = SystemConfigReader()
database_options = scr.get_all_values_from_section('Database')
database = DatabaseManagerMongo(**database_options)
super(ExportdThread, self).__init__()
self.job = None
self.job_id = int(event.get_param("id"))
self.type_id = int(
event.get_param("type_id")) if event.get_param("type_id") else None
self.user_id = int(event.get_param("user_id"))
self.event = event
self.is_active = state
self.exception_handling = None
self.object_manager = CmdbObjectManager(database_manager=database)
self.log_manager = ExportdLogManager(database_manager=database)
self.exportd_job_manager = ExportdJobManagement(
database_manager=database)
self.user_manager = UserManager(database_manager=database)
def _run(self):
# get queue for sending events
event_queue = self._event_manager.get_send_queue()
database_manager = DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database'))
# get WSGI app
app = DispatcherMiddleware(app=create_app(),
mounts={
'/docs':
create_docs_server(),
'/rest':
create_rest_api(database_manager,
event_queue)
})
# get gunicorn options
options = SystemConfigReader().get_all_values_from_section('WebServer')
# start gunicorn as own process
webserver = HTTPServer(app, options)
self.__webserver_proc = multiprocessing.Process(target=webserver.run)
self.__webserver_proc.start()
self.__webserver_proc.join()
def authenticate(self, user_name: str, password: str, **kwargs) -> UserModel:
__dbm = DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database')
)
__scm = SecurityManager(__dbm)
__user_manager = UserManager(__dbm)
LOGGER.info(f'[LocalAuthenticationProvider] Try login for user {user_name}')
try:
user: UserModel = __user_manager.get_by({'user_name': user_name})
except ManagerGetError as umge:
raise AuthenticationError(LocalAuthenticationProvider.get_name(), umge.message)
login_pass = __scm.generate_hmac(password)
if login_pass == user.password:
return user
raise AuthenticationError(LocalAuthenticationProvider.get_name(), 'UserModel not exists')
def get_config_information(request_user: User):
ssc = SystemConfigReader()
config_dict = {'path': ssc.config_file, 'properties': []}
ssc = SystemConfigReader()
for section in ssc.get_sections():
section_values = []
for key, value in ssc.get_all_values_from_section(section).items():
section_values.append([key, value])
config_dict['properties'].append([section, section_values])
if len(config_dict) < 1:
return make_response(config_dict, 204)
return make_response(config_dict)
class KeyHolder:
def __init__(self, key_directory=None):
"""
Args:
key_directory: key based directory
"""
self.scr = SystemConfigReader()
self.__dbm = DatabaseManagerMongo(
**self.scr.get_all_values_from_section('Database'))
self.ssr = SystemSettingsReader(self.__dbm)
self.rsa_public = self.get_public_key()
self.rsa_private = self.get_private_key()
def get_public_key(self):
return self.ssr.get_value('asymmetric_key', 'security')['public']
def get_private_key(self):
return self.ssr.get_value('asymmetric_key', 'security')['private']
class SetupRoutine:
class SetupStatus(Enum):
NOT = 0
RUNNING = 1
ERROR = 2
FINISHED = 3
def __init__(self, dbm: DatabaseManagerMongo):
self.status = SetupRoutine.SetupStatus.NOT
# check if settings are loaded
self.setup_system_config_reader = SystemConfigReader()
self.setup_database_manager = dbm
system_config_reader_status = self.setup_system_config_reader.status()
if system_config_reader_status is not True:
self.status = SetupRoutine.SetupStatus.ERROR
raise RuntimeError(
f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \
system config reader status: {system_config_reader_status}'
)
def get_setup_status(self):
return self.status
def setup(self) -> SetupStatus:
LOGGER.info('SETUP ROUTINE: STARTED...')
self.status = SetupRoutine.SetupStatus.RUNNING
# check database
if not self.__check_database():
self.status = SetupRoutine.SetupStatus.ERROR
raise RuntimeError(
'The database managers could not be initialized. Perhaps the database cannot be reached, \
or the database was already initialized.')
if self.__is_database_empty():
# init database
try:
self.__init_database()
except Exception as err:
self.status = SetupRoutine.SetupStatus.ERROR
raise RuntimeError(
f'Something went wrong during the initialization of the database. \n Error: {err}'
)
# generate keys
LOGGER.info('SETUP ROUTINE: Generate rsa key pair')
try:
self.init_keys()
except Exception as err:
self.status = SetupRoutine.SetupStatus.ERROR
raise RuntimeError(
f'Something went wrong during the generation of the rsa keypair. \n Error: {err}'
)
# create user management
LOGGER.info('SETUP ROUTINE: UserModel management')
try:
self.__create_user_management()
except Exception as err:
self.status = SetupRoutine.SetupStatus.ERROR
raise RuntimeError(
f'Something went wrong during the generation of the user management. \n Error: {err}'
)
# create version updater settings
try:
from cmdb.updater import UpdaterModule
from cmdb.updater.updater_settings import UpdateSettings
from cmdb.utils.system_reader import SystemSettingsReader
from cmdb.utils.system_writer import SystemSettingsWriter
system_setting_writer: SystemSettingsWriter = SystemSettingsWriter(
self.setup_database_manager)
updater_setting_instance = UpdateSettings(
**UpdaterModule.get_last_version())
system_setting_writer.write(
_id='updater', data=updater_setting_instance.__dict__)
except Exception as err:
self.status = SetupRoutine.SetupStatus.ERROR
raise RuntimeError(
f'Something went wrong during the generation of the updater module. \n Error: {err}'
)
self.status = SetupRoutine.SetupStatus.FINISHED
LOGGER.info('SETUP ROUTINE: FINISHED!')
return self.status
def init_keys(self):
from cmdb.security.key.generator import KeyGenerator
kg = KeyGenerator()
LOGGER.info('KEY ROUTINE: Generate RSA keypair')
kg.generate_rsa_keypair()
LOGGER.info('KEY ROUTINE: Generate aes key')
kg.generate_symmetric_aes_key()
self.__check_database()
from cmdb.user_management.managers.user_manager import UserManager, UserModel
from cmdb.security.security import SecurityManager
scm = SecurityManager(self.setup_database_manager)
usm = UserManager(self.setup_database_manager)
try:
admin_user: UserModel = usm.get(1)
LOGGER.warning('KEY ROUTINE: Admin user detected')
LOGGER.info(
f'KEY ROUTINE: Enter new password for user: {admin_user.user_name}'
)
admin_pass = str(input('New admin password: '******'KEY ROUTINE: Password was updated for user: {admin_user.user_name}'
)
except Exception as ex:
LOGGER.info(
f'KEY ROUTINE: Password was updated for user failed: {ex}')
LOGGER.info('KEY ROUTINE: FINISHED')
def __create_user_management(self):
from cmdb.user_management.models.user import UserModel
from cmdb.user_management.managers.user_manager import UserManager
from cmdb.user_management.managers.group_manager import GroupManager
from cmdb.user_management import __FIXED_GROUPS__
from cmdb.security.security import SecurityManager
scm = SecurityManager(self.setup_database_manager)
group_manager = GroupManager(self.setup_database_manager)
user_manager = UserManager(self.setup_database_manager)
for group in __FIXED_GROUPS__:
group_manager.insert(group)
# setting the initial user to admin/admin as default
admin_name = 'admin'
admin_pass = '******'
import datetime
admin_user = UserModel(
public_id=1,
user_name=admin_name,
active=True,
group_id=__FIXED_GROUPS__[0].get_public_id(),
registration_time=datetime.datetime.now(),
password=scm.generate_hmac(admin_pass),
)
user_manager.insert(admin_user)
return True
def __check_database(self):
LOGGER.info('SETUP ROUTINE: Checking database connection')
from cmdb.database.errors.connection_errors import ServerTimeoutError
try:
connection_test = self.setup_database_manager.connector.is_connected(
)
except ServerTimeoutError:
connection_test = False
LOGGER.info(
f'SETUP ROUTINE: Database connection status {connection_test}')
return connection_test
def __is_database_empty(self) -> bool:
return not self.setup_database_manager.connector.database.list_collection_names(
)
def __init_database(self):
database_name = self.setup_system_config_reader.get_value(
'database_name', 'Database')
LOGGER.info(f'SETUP ROUTINE: initialize database {database_name}')
# delete database
try:
self.setup_database_manager.drop_database(database_name)
except DatabaseNotExists as err:
LOGGER.error(err.message)
# create new database
self.setup_database_manager.create_database(database_name)
# generate collections
# framework collections
from cmdb.framework import __COLLECTIONS__ as FRAMEWORK_CLASSES
for collection in FRAMEWORK_CLASSES:
self.setup_database_manager.create_collection(
collection.COLLECTION)
# set unique indexes
self.setup_database_manager.create_indexes(
collection.COLLECTION, collection.get_index_keys())
# user management collections
from cmdb.user_management import __COLLECTIONS__ as USER_MANAGEMENT_COLLECTION
for collection in USER_MANAGEMENT_COLLECTION:
self.setup_database_manager.create_collection(
collection.COLLECTION)
# set unique indexes
self.setup_database_manager.create_indexes(
collection.COLLECTION, collection.get_index_keys())
# ExportdJob management collections
from cmdb.exportd import __COLLECTIONS__ as JOB_MANAGEMENT_COLLECTION
for collection in JOB_MANAGEMENT_COLLECTION:
self.setup_database_manager.create_collection(
collection.COLLECTION)
# set unique indexes
self.setup_database_manager.create_indexes(
collection.COLLECTION, collection.get_index_keys())
LOGGER.info('SETUP ROUTINE: initialize finished')
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import logging
import cmdb
from cmdb.interface.cmdb_app import BaseCmdbApp
from cmdb.interface.config import app_config
from cmdb.interface.docs.doc_routes import doc_pages
from cmdb.utils.error import CMDBError
LOGGER = logging.getLogger(__name__)
from cmdb.utils.system_config import SystemConfigReader
system_config_reader = SystemConfigReader()
def create_docs_server(event_queue):
# Create managers
from cmdb.data_storage.database_manager import DatabaseManagerMongo
app_database = DatabaseManagerMongo(
**system_config_reader.get_all_values_from_section('Database')
)
app = BaseCmdbApp(__name__, app_database)
if cmdb.__MODE__ == 'DEBUG':
config = app_config['development']
config.APPLICATION_ROOT = '/docs/'
app.config.from_object(config)
def test_config_less_instance(self):
from cmdb.utils.system_config import SystemConfigReader
system_config_reader = SystemConfigReader()
assert system_config_reader.config_file_less is True
def config_reader(pytestconfig: Config) -> SystemConfigReader:
config_path = pytestconfig.getoption('config-path',
'../../etc/cmdb_test.conf')
return SystemConfigReader.from_full_path(config_path)
from flask import abort, Response
from cmdb.data_storage.database_manager import DatabaseManagerMongo
from cmdb.framework.cmdb_errors import ObjectNotFoundError, TypeNotFoundError
from cmdb.file_export.export_types import ExportType
from cmdb.framework.cmdb_object_manager import CmdbObjectManager
try:
from cmdb.utils.error import CMDBError
except ImportError:
CMDBError = Exception
from cmdb.utils.system_config import SystemConfigReader
object_manager = CmdbObjectManager(database_manager=DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database')))
class FileExporter:
@staticmethod
def get_filetypes():
filetypes = [
"CsvExportType", "JsonExportType", "XlsxExportType",
"XmlExportType"
]
return filetypes
def __init__(self, object_type, export_type: ExportType, public_id: str,
**kwargs):
"""init of FileExporter
class EventSenderAmqp(threading.Thread):
"""EventSender part of the EventManagerAmqp
This part of the EventManagerAmqp is responsible for sending events
to the message broker using the protocol AMQP.
"""
def __init__(self, message_queue, flag_shutdown, process_id=None):
"""Creates an instance of EventSenderAmqp
Args:
message_queue(queue.Queue): handler of a queue for sending events
flag_shutdown(threading.Event): flag for handling shutdown
process_id(str): process identifier (process name)
"""
super(EventSenderAmqp, self).__init__()
self.__queue = message_queue
self.__flag_shutdown = flag_shutdown
self.__process_id = process_id
# get configuration
self.__config_mq = SystemConfigReader().get_all_values_from_section(
'MessageQueueing')
self.__config_host = self.__config_mq.get("host", "127.0.0.1")
self.__config_port = self.__config_mq.get("port", "5672")
self.__config_username = self.__config_mq.get("username", "guest")
self.__config_password = self.__config_mq.get("password", "guest")
self.__config_exchange = self.__config_mq.get("exchange",
"datagerry.eventbus")
self.__config_retries = int(
self.__config_mq.get("connection_attempts", "5"))
self.__config_retrydelay = int(self.__config_mq.get(
"retry_delay", "6"))
self.__config_tls = False
if self.__config_mq.get("use_tls",
"False") in ("true", "True", "1", "yes"):
self.__config_tls = True
# define variables
self.__connection = None
self.__channel = None
def __init_connection(self):
"""create a connection to message broker"""
try:
credentials = pika.credentials.PlainCredentials(
self.__config_username, self.__config_password)
self.__connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=self.__config_host,
port=self.__config_port,
connection_attempts=self.__config_retries,
retry_delay=self.__config_retrydelay,
credentials=credentials,
ssl=self.__config_tls))
self.__channel = self.__connection.channel()
self.__channel.exchange_declare(exchange=self.__config_exchange,
exchange_type="topic")
except pika.exceptions.AMQPConnectionError:
LOGGER.error("{}: EventSenderAmqp connection error".format(
self.__process_id))
self.__flag_shutdown.set()
def run(self):
"""run the event sender"""
# init connection to broker
self.__init_connection()
# check queue for new events
while not self.__flag_shutdown.is_set():
try:
try:
event = self.__queue.get(block=True, timeout=2)
event_type = event.get_type()
event_serialized = event.json_repr()
self.__channel.basic_publish(
exchange=self.__config_exchange,
routing_key=event_type,
body=event_serialized)
except queue.Empty:
self.__connection.process_data_events()
# handle AMQP connection errors
except pika.exceptions.AMQPConnectionError:
LOGGER.warning(
"connection to broker lost, try to reconnect...")
self.__init_connection()
class EventReceiverAmqp(threading.Thread):
"""EventReceiver part of the EventManagerAmqp
This part of the EventManagerAmqp is responsible for receiving events
from the message broker using the protocol AMQP. Only configured event
types were processed (e.g. cmdb.core.objects.added)
"""
def __init__(self,
receiver_callback,
flag_shutdown,
process_id=None,
event_types=["#"]):
"""Creates an instance of EventReceiverAmqp
Args:
receiver_callback(func): callback function for processing
received events
flag_shutdown(threading.Event): flag for handling shutdown
process_id(str): process identifier (process name)
event_types(list): list of event types, that will be processed
"""
super(EventReceiverAmqp, self).__init__()
self.__receiver_callback = receiver_callback
self.__flag_shutdown = flag_shutdown
self.__process_id = process_id
self.__event_types = event_types
# get configuration
self.__config_mq = SystemConfigReader().get_all_values_from_section(
'MessageQueueing')
self.__config_host = self.__config_mq.get("host", "127.0.0.1")
self.__config_port = self.__config_mq.get("port", "5672")
self.__config_username = self.__config_mq.get("username", "guest")
self.__config_password = self.__config_mq.get("password", "guest")
self.__config_exchange = self.__config_mq.get("exchange",
"datagerry.eventbus")
self.__config_retries = int(
self.__config_mq.get("connection_attempts", "5"))
self.__config_retrydelay = int(self.__config_mq.get(
"retry_delay", "6"))
self.__config_tls = False
if self.__config_mq.get("use_tls",
"False") in ("true", "True", "1", "yes"):
self.__config_tls = True
# define variables
self.__connection = None
self.__channel = None
def __process_event_cb(self, ch, method, properties, body):
"""event processing
this callback function is executed, if an event was reveived.
Args:
ch: AMQP channel
method: AMQP method
properties: AMQP properties
body: AMQP message body
"""
event = Event.create_event(body)
# allow None values, if event receiving should be ignored
if self.__receiver_callback:
self.__receiver_callback(event)
def __check_shutdown_flag(self):
"""check, if the shutdown flag was set"""
# reinstall shutdown check
self.__connection.add_timeout(2, self.__check_shutdown_flag)
# check shutdown flag
if self.__flag_shutdown.is_set():
self.__channel.stop_consuming()
self.__connection.close()
def __init_connection(self):
"""create a connection to message broker"""
try:
# init connection to broker
credentials = pika.credentials.PlainCredentials(
self.__config_username, self.__config_password)
self.__connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=self.__config_host,
port=self.__config_port,
connection_attempts=self.__config_retries,
retry_delay=self.__config_retrydelay,
credentials=credentials,
ssl=self.__config_tls))
self.__channel = self.__connection.channel()
self.__channel.exchange_declare(exchange=self.__config_exchange,
exchange_type="topic")
queue_name = ""
if self.__process_id:
queue_name = "{}.{}".format(self.__config_exchange,
self.__process_id)
queue_declare_result = self.__channel.queue_declare(
queue=queue_name, exclusive=True)
queue_handler = queue_declare_result.method.queue
for event_type in self.__event_types:
self.__channel.queue_bind(exchange=self.__config_exchange,
queue=queue_handler,
routing_key=event_type)
# register callback function for event handling
self.__channel.basic_consume(self.__process_event_cb,
queue=queue_handler,
no_ack=True)
except AMQPConnectionError:
LOGGER.error("{}: EventReceiverAmqp connection error".format(
self.__process_id))
self.__flag_shutdown.set()
def run(self):
"""run the event receiver"""
# init connection to broker
self.__init_connection()
while not self.__flag_shutdown.is_set():
try:
# start handling events
self.__connection.add_timeout(2, self.__check_shutdown_flag)
self.__channel.start_consuming()
# handle AMQP connection errors
except AMQPConnectionError:
LOGGER.warning(
"connection to broker lost, try to reconnect...")
self.__init_connection()
def main(args):
"""
Default application start function
Args:
args: start-options
"""
LOGGER.info("DATAGERRY starting...")
if args.debug:
_activate_debug()
_init_config_reader(args.config_file)
from cmdb.data_storage.database_connection import DatabaseConnectionError
# create / check connection database manager
dbm = None
try:
dbm = _check_database()
if not dbm:
raise DatabaseConnectionError('')
LOGGER.info("Database connection established.")
except CMDBError as conn_error:
LOGGER.critical(conn_error.message)
exit(1)
# check db-settings and run update if needed
if args.start:
from cmdb.__check__ import CheckRoutine
check_routine = CheckRoutine(dbm)
# check db-settings
try:
check_status = check_routine.checker()
except Exception as err:
LOGGER.error(err)
check_status = check_routine.get_check_status()
LOGGER.error(
f'The check did not go through as expected. Please run an update. \n Error: {err}'
)
if check_status == CheckRoutine.CheckStatus.HAS_UPDATES:
# run update
from cmdb.__update__ import UpdateRoutine
update_routine = UpdateRoutine()
try:
update_status = update_routine.start_update()
except RuntimeError as err:
LOGGER.error(err)
update_status = update_routine.get_updater_status()
LOGGER.warning(
f'The update did not go through as expected - Status {update_status}'
)
if update_status == UpdateRoutine.UpateStatus.FINISHED:
check_status = CheckRoutine.CheckStatus.FINISHED
else:
exit(1)
if check_status == CheckRoutine.CheckStatus.FINISHED:
# run setup if needed
from cmdb.__setup__ import SetupRoutine
setup_routine = SetupRoutine(dbm)
try:
setup_status = setup_routine.setup()
except RuntimeError as err:
LOGGER.error(err)
setup_status = setup_routine.get_setup_status()
LOGGER.warning(
f'The setup did not go through as expected - Status {setup_status}'
)
if setup_status == SetupRoutine.SetupStatus.FINISHED:
pass
else:
exit(1)
else:
pass
if args.keys:
from cmdb.__setup__ import SetupRoutine
setup_routine = SetupRoutine(dbm)
setup_status = None
try:
setup_routine.init_keys()
except RuntimeError as err:
LOGGER.error(err)
setup_status = setup_routine.get_setup_status()
LOGGER.warning(
f'The key generation did not go through as expected - Status {setup_status}'
)
if setup_status == SetupRoutine.SetupStatus.FINISHED:
exit(0)
else:
exit(1)
if args.test_data:
_activate_debug()
from cmdb.utils.data_factory import DataFactory
ssc = SystemConfigReader()
database_options = ssc.get_all_values_from_section('Database')
dbm = DatabaseManagerMongo(**database_options)
db_name = dbm.get_database_name()
LOGGER.warning(f'Inserting test-data into: {db_name}')
try:
factory = DataFactory(database_manager=dbm)
ack = factory.insert_data()
LOGGER.warning("Test-data was successfully added".format(
dbm.get_database_name()))
if len(ack) > 0:
LOGGER.critical(
"Error while inserting test-data: {} - dropping database".
format(ack))
dbm.drop(db_name) # cleanup database
except (Exception, CMDBError) as e:
import traceback
traceback.print_tb(e.__traceback__)
dbm.drop(db_name) # cleanup database
exit(1)
if args.start:
_start_app()
sleep(0.2) # prevent logger output
LOGGER.info("DATAGERRY successfully started")
class UpdateRoutine:
class UpateStatus(Enum):
NOT = 0
RUNNING = 1
ERROR = 2
FINISHED = 3
def __init__(self):
self.status = UpdateRoutine.UpateStatus.NOT
# check if settings are loaded
from cmdb.utils.system_config import SystemConfigReader
self.setup_system_config_reader = SystemConfigReader()
system_config_reader_status = self.setup_system_config_reader.status()
if system_config_reader_status is not True:
self.status = UpdateRoutine.UpateStatus.ERROR
raise RuntimeError(
f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \
system config reader status: {system_config_reader_status}')
def get_updater_status(self):
return self.status
def __check_database(self):
LOGGER.info('SETUP ROUTINE: Checking database connection')
from cmdb.data_storage.database_manager import DatabaseManagerMongo
from cmdb.data_storage.database_connection import ServerTimeoutError
try:
self.setup_database_manager = DatabaseManagerMongo(
**self.setup_system_config_reader.get_all_values_from_section('Database')
)
connection_test = self.setup_database_manager.connector.is_connected()
except ServerTimeoutError:
connection_test = False
LOGGER.info(f'SETUP ROUTINE: Database connection status {connection_test}')
return connection_test
def __is_database_empty(self) -> bool:
return not self.setup_database_manager.connector.database.list_collection_names()
def start_update(self):
LOGGER.info('UPDATE ROUTINE: Update database collection')
self.status = UpdateRoutine.UpateStatus.RUNNING
# check database
if not self.__check_database():
self.status = UpdateRoutine.UpateStatus.ERROR
raise RuntimeError(
'The database manager could not be initialized. Perhaps the database cannot be reached, \
or the database was already initialized.'
)
if not self.__is_database_empty():
self.update_database_collection()
self.update_db_version()
else:
LOGGER.info('UPDATE ROUTINE: The update is faulty because no collection was detected.')
LOGGER.info('UPDATE ROUTINE: Update database collection finished.')
self.status = UpdateRoutine.UpateStatus.FINISHED
LOGGER.info('UPDATE ROUTINE: FINISHED!')
return self.status
def update_database_collection(self):
from cmdb.framework import __COLLECTIONS__ as FRAMEWORK_CLASSES
from cmdb.user_management import __COLLECTIONS__ as USER_MANAGEMENT_COLLECTION
from cmdb.exportd import __COLLECTIONS__ as JOB_MANAGEMENT_COLLECTION
# update database collections
try:
detected_database = self.setup_database_manager.connector.database
# update collections
# framework collections
for collection in FRAMEWORK_CLASSES:
try:
detected_database.validate_collection(collection.COLLECTION)['valid']
except:
self.setup_database_manager.create_collection(collection.COLLECTION)
# set unique indexes
self.setup_database_manager.create_indexes(collection.COLLECTION, collection.get_index_keys())
LOGGER.info(f'UPDATE ROUTINE: Database collection {collection.COLLECTION} was created.')
# user management collections
for collection in USER_MANAGEMENT_COLLECTION:
try:
detected_database.validate_collection(collection.COLLECTION)['valid']
except:
self.setup_database_manager.create_collection(collection.COLLECTION)
# set unique indexes
self.setup_database_manager.create_indexes(collection.COLLECTION, collection.get_index_keys())
LOGGER.info(f'UPDATE ROUTINE: Database collection {collection.COLLECTION} was created.')
# exportdJob management collections
for collection in JOB_MANAGEMENT_COLLECTION:
try:
detected_database.validate_collection(collection.COLLECTION)['valid']
except:
self.setup_database_manager.create_collection(collection.COLLECTION)
# set unique indexes
self.setup_database_manager.create_indexes(collection.COLLECTION,
collection.get_index_keys())
LOGGER.info(f'UPDATE ROUTINE: Database collection {collection.COLLECTION} was created.')
except Exception as ex:
LOGGER.info(f'UPDATE ROUTINE: Database collection validation failed: {ex}')
def update_db_version(self):
# update version updater settings
try:
updater_settings_values = UpdaterModule.__DEFAULT_SETTINGS__
ssr = SystemSettingsReader(self.setup_database_manager)
try:
updater_settings_values = ssr.get_all_values_from_section('updater')
updater_setting_instance = UpdateSettings(**updater_settings_values)
except Exception:
# create updater section if not exist
system_setting_writer: SystemSettingsWriter = SystemSettingsWriter(self.setup_database_manager)
updater_setting_instance = UpdateSettings(**updater_settings_values)
system_setting_writer.write(_id='updater', data=updater_setting_instance.__dict__)
# start running update files
updater_setting_instance.run_updates(updater_settings_values.get('version'), ssr)
except Exception as err:
self.status = UpdateRoutine.UpateStatus.ERROR
raise RuntimeError(
f'Something went wrong during the generation of the updater module. \n Error: {err}'
)