def _check_database():
"""
Checks whether the specified connection of the configuration is reachable.
Returns: Datebase if response
"""
from cmdb.database.errors.connection_errors import ServerTimeoutError
ssc = SystemConfigReader()
LOGGER.info(f'Checking database connection with {ssc.config_name} data')
database_options = ssc.get_all_values_from_section('Database')
dbm = DatabaseManagerMongo(**database_options)
try:
connection_test = dbm.connector.is_connected()
except ServerTimeoutError:
connection_test = False
LOGGER.debug(f'Database status is {connection_test}')
if connection_test is True:
return dbm
retries = 0
while retries < 3:
retries += 1
LOGGER.warning(
f'Retry {retries}: Checking database connection with {SystemConfigReader.DEFAULT_CONFIG_NAME} data'
)
connection_test = dbm.connector.is_connected()
if connection_test:
return dbm
return None
def __init__(self, key_directory=None):
self.scr = SystemConfigReader()
ssc = SystemConfigReader()
database_options = ssc.get_all_values_from_section('Database')
self.__dbm = DatabaseManagerMongo(**database_options)
self.ssw = SystemSettingsWriter(self.__dbm)
self.key_directory = key_directory or SystemConfigReader.DEFAULT_CONFIG_LOCATION + "/keys"
def __init__(self):
scr = SystemConfigReader()
self.database_manager = DatabaseManagerMongo(
**scr.get_all_values_from_section('Database'))
self.object_manager = CmdbObjectManager(
database_manager=self.database_manager)
self.log_manager = ExportdLogManager(
database_manager=self.database_manager)
super().__init__(self.database_manager)
def get_config_information(request_user: User):
ssc = SystemConfigReader()
config_dict = {'path': ssc.config_file, 'properties': []}
ssc = SystemConfigReader()
for section in ssc.get_sections():
section_values = []
for key, value in ssc.get_all_values_from_section(section).items():
section_values.append([key, value])
config_dict['properties'].append([section, section_values])
if len(config_dict) < 1:
return make_response(config_dict, 204)
return make_response(config_dict)
def __init__(self, job: ExportdJob):
self.job = job
self.exportvars = self.__get_exportvars()
self.destinations = self.__get__destinations()
scr = SystemConfigReader()
database_manager = DatabaseManagerMongo(
**scr.get_all_values_from_section('Database'))
self.__object_manager = CmdbObjectManager(
database_manager=database_manager)
self.log_manager = ExportdLogManager(database_manager=database_manager)
self.sources = self.__get_sources()
super(ExportdManagerBase, self).__init__(database_manager)
def __init__(self, event, state=False):
super(ExportdThread, self).__init__()
self.job = None
self.job_id = event.get_param("id")
self.type_id = event.get_param("type_id")
self.user_id = event.get_param("user_id")
self.event = event
self.is_active = state
self.exception_handling = None
scr = SystemConfigReader()
database_options = scr.get_all_values_from_section('Database')
self.__dbm = DatabaseManagerMongo(**database_options)
self.log_manager = ExportdLogManager(database_manager=self.__dbm)
self.exportd_job_manager = ExportdJobManagement(
database_manager=self.__dbm)
self.user_manager = UserManager(database_manager=self.__dbm)
class KeyHolder:
def __init__(self, key_directory=None):
"""
Args:
key_directory: key based directory
"""
self.scr = SystemConfigReader()
self.__dbm = DatabaseManagerMongo(
**self.scr.get_all_values_from_section('Database'))
self.ssr = SystemSettingsReader(self.__dbm)
self.rsa_public = self.get_public_key()
self.rsa_private = self.get_private_key()
def get_public_key(self):
return self.ssr.get_value('asymmetric_key', 'security')['public']
def get_private_key(self):
return self.ssr.get_value('asymmetric_key', 'security')['private']
def __init__(self, event: Event, state: bool = False):
scr = SystemConfigReader()
database_options = scr.get_all_values_from_section('Database')
database = DatabaseManagerMongo(**database_options)
super(ExportdThread, self).__init__()
self.job = None
self.job_id = int(event.get_param("id"))
self.type_id = int(
event.get_param("type_id")) if event.get_param("type_id") else None
self.user_id = int(event.get_param("user_id"))
self.event = event
self.is_active = state
self.exception_handling = None
self.object_manager = CmdbObjectManager(database_manager=database)
self.log_manager = ExportdLogManager(database_manager=database)
self.exportd_job_manager = ExportdJobManagement(
database_manager=database)
self.user_manager = UserManager(database_manager=database)
class UpdateRoutine:
class UpateStatus(Enum):
NOT = 0
RUNNING = 1
ERROR = 2
FINISHED = 3
def __init__(self):
self.status = UpdateRoutine.UpateStatus.NOT
# check if settings are loaded
from cmdb.utils.system_config import SystemConfigReader
self.setup_system_config_reader = SystemConfigReader()
system_config_reader_status = self.setup_system_config_reader.status()
if system_config_reader_status is not True:
self.status = UpdateRoutine.UpateStatus.ERROR
raise RuntimeError(
f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \
system config reader status: {system_config_reader_status}')
def get_updater_status(self):
return self.status
def __check_database(self):
LOGGER.info('SETUP ROUTINE: Checking database connection')
from cmdb.data_storage.database_manager import DatabaseManagerMongo
from cmdb.data_storage.database_connection import ServerTimeoutError
try:
self.setup_database_manager = DatabaseManagerMongo(
**self.setup_system_config_reader.get_all_values_from_section('Database')
)
connection_test = self.setup_database_manager.connector.is_connected()
except ServerTimeoutError:
connection_test = False
LOGGER.info(f'SETUP ROUTINE: Database connection status {connection_test}')
return connection_test
def __is_database_empty(self) -> bool:
return not self.setup_database_manager.connector.database.list_collection_names()
def start_update(self):
LOGGER.info('UPDATE ROUTINE: Update database collection')
self.status = UpdateRoutine.UpateStatus.RUNNING
# check database
if not self.__check_database():
self.status = UpdateRoutine.UpateStatus.ERROR
raise RuntimeError(
'The database manager could not be initialized. Perhaps the database cannot be reached, \
or the database was already initialized.'
)
if not self.__is_database_empty():
self.update_database_collection()
self.update_db_version()
else:
LOGGER.info('UPDATE ROUTINE: The update is faulty because no collection was detected.')
LOGGER.info('UPDATE ROUTINE: Update database collection finished.')
self.status = UpdateRoutine.UpateStatus.FINISHED
LOGGER.info('UPDATE ROUTINE: FINISHED!')
return self.status
def update_database_collection(self):
from cmdb.framework import __COLLECTIONS__ as FRAMEWORK_CLASSES
from cmdb.user_management import __COLLECTIONS__ as USER_MANAGEMENT_COLLECTION
from cmdb.exportd import __COLLECTIONS__ as JOB_MANAGEMENT_COLLECTION
# update database collections
try:
detected_database = self.setup_database_manager.connector.database
# update collections
# framework collections
for collection in FRAMEWORK_CLASSES:
try:
detected_database.validate_collection(collection.COLLECTION)['valid']
except:
self.setup_database_manager.create_collection(collection.COLLECTION)
# set unique indexes
self.setup_database_manager.create_indexes(collection.COLLECTION, collection.get_index_keys())
LOGGER.info(f'UPDATE ROUTINE: Database collection {collection.COLLECTION} was created.')
# user management collections
for collection in USER_MANAGEMENT_COLLECTION:
try:
detected_database.validate_collection(collection.COLLECTION)['valid']
except:
self.setup_database_manager.create_collection(collection.COLLECTION)
# set unique indexes
self.setup_database_manager.create_indexes(collection.COLLECTION, collection.get_index_keys())
LOGGER.info(f'UPDATE ROUTINE: Database collection {collection.COLLECTION} was created.')
# exportdJob management collections
for collection in JOB_MANAGEMENT_COLLECTION:
try:
detected_database.validate_collection(collection.COLLECTION)['valid']
except:
self.setup_database_manager.create_collection(collection.COLLECTION)
# set unique indexes
self.setup_database_manager.create_indexes(collection.COLLECTION,
collection.get_index_keys())
LOGGER.info(f'UPDATE ROUTINE: Database collection {collection.COLLECTION} was created.')
except Exception as ex:
LOGGER.info(f'UPDATE ROUTINE: Database collection validation failed: {ex}')
def update_db_version(self):
# update version updater settings
try:
updater_settings_values = UpdaterModule.__DEFAULT_SETTINGS__
ssr = SystemSettingsReader(self.setup_database_manager)
try:
updater_settings_values = ssr.get_all_values_from_section('updater')
updater_setting_instance = UpdateSettings(**updater_settings_values)
except Exception:
# create updater section if not exist
system_setting_writer: SystemSettingsWriter = SystemSettingsWriter(self.setup_database_manager)
updater_setting_instance = UpdateSettings(**updater_settings_values)
system_setting_writer.write(_id='updater', data=updater_setting_instance.__dict__)
# start running update files
updater_setting_instance.run_updates(updater_settings_values.get('version'), ssr)
except Exception as err:
self.status = UpdateRoutine.UpateStatus.ERROR
raise RuntimeError(
f'Something went wrong during the generation of the updater module. \n Error: {err}'
)
def main(args):
"""
Default application start function
Args:
args: start-options
"""
LOGGER.info("DATAGERRY starting...")
if args.debug:
_activate_debug()
_init_config_reader(args.config_file)
from cmdb.data_storage.database_connection import DatabaseConnectionError
# create / check connection database manager
dbm = None
try:
dbm = _check_database()
if not dbm:
raise DatabaseConnectionError('')
LOGGER.info("Database connection established.")
except CMDBError as conn_error:
LOGGER.critical(conn_error.message)
exit(1)
# check db-settings and run update if needed
if args.start:
from cmdb.__check__ import CheckRoutine
check_routine = CheckRoutine(dbm)
# check db-settings
try:
check_status = check_routine.checker()
except Exception as err:
LOGGER.error(err)
check_status = check_routine.get_check_status()
LOGGER.error(
f'The check did not go through as expected. Please run an update. \n Error: {err}'
)
if check_status == CheckRoutine.CheckStatus.HAS_UPDATES:
# run update
from cmdb.__update__ import UpdateRoutine
update_routine = UpdateRoutine()
try:
update_status = update_routine.start_update()
except RuntimeError as err:
LOGGER.error(err)
update_status = update_routine.get_updater_status()
LOGGER.warning(
f'The update did not go through as expected - Status {update_status}'
)
if update_status == UpdateRoutine.UpateStatus.FINISHED:
check_status = CheckRoutine.CheckStatus.FINISHED
else:
exit(1)
if check_status == CheckRoutine.CheckStatus.FINISHED:
# run setup if needed
from cmdb.__setup__ import SetupRoutine
setup_routine = SetupRoutine(dbm)
try:
setup_status = setup_routine.setup()
except RuntimeError as err:
LOGGER.error(err)
setup_status = setup_routine.get_setup_status()
LOGGER.warning(
f'The setup did not go through as expected - Status {setup_status}'
)
if setup_status == SetupRoutine.SetupStatus.FINISHED:
pass
else:
exit(1)
else:
pass
if args.keys:
from cmdb.__setup__ import SetupRoutine
setup_routine = SetupRoutine(dbm)
setup_status = None
try:
setup_routine.init_keys()
except RuntimeError as err:
LOGGER.error(err)
setup_status = setup_routine.get_setup_status()
LOGGER.warning(
f'The key generation did not go through as expected - Status {setup_status}'
)
if setup_status == SetupRoutine.SetupStatus.FINISHED:
exit(0)
else:
exit(1)
if args.test_data:
_activate_debug()
from cmdb.utils.data_factory import DataFactory
ssc = SystemConfigReader()
database_options = ssc.get_all_values_from_section('Database')
dbm = DatabaseManagerMongo(**database_options)
db_name = dbm.get_database_name()
LOGGER.warning(f'Inserting test-data into: {db_name}')
try:
factory = DataFactory(database_manager=dbm)
ack = factory.insert_data()
LOGGER.warning("Test-data was successfully added".format(
dbm.get_database_name()))
if len(ack) > 0:
LOGGER.critical(
"Error while inserting test-data: {} - dropping database".
format(ack))
dbm.drop(db_name) # cleanup database
except (Exception, CMDBError) as e:
import traceback
traceback.print_tb(e.__traceback__)
dbm.drop(db_name) # cleanup database
exit(1)
if args.start:
_start_app()
sleep(0.2) # prevent logger output
LOGGER.info("DATAGERRY successfully started")
def create_rest_api(event_queue):
from cmdb.interface.config import app_config
from cmdb.utils.system_config import SystemConfigReader
system_config_reader = SystemConfigReader()
# Create managers
from cmdb.data_storage.database_manager import DatabaseManagerMongo
app_database = DatabaseManagerMongo(
**system_config_reader.get_all_values_from_section('Database'))
object_manager = CmdbObjectManager(database_manager=app_database,
event_queue=event_queue)
log_manager = CmdbLogManager(database_manager=app_database)
security_manager = SecurityManager(database_manager=app_database)
user_manager = UserManager(database_manager=app_database)
exportd_job_manager = ExportdJobManagement(database_manager=app_database,
event_queue=event_queue)
exportd_log_manager = ExportdLogManager(database_manager=app_database)
media_file_manager = MediaFileManagement(database_manager=app_database)
docapi_tpl_manager = DocapiTemplateManager(database_manager=app_database)
# Create APP
from cmdb.interface.cmdb_app import BaseCmdbApp
app = BaseCmdbApp(__name__,
database_manager=app_database,
docapi_tpl_manager=docapi_tpl_manager,
media_file_manager=media_file_manager,
exportd_manager=exportd_job_manager,
exportd_log_manager=exportd_log_manager,
object_manager=object_manager,
log_manager=log_manager,
user_manager=user_manager,
security_manager=security_manager)
app.url_map.strict_slashes = True
# Import App Extensions
from flask_cors import CORS
CORS(app, expose_headers=['X-API-Version', 'X-Total-Count'])
import cmdb
if cmdb.__MODE__ == 'DEBUG':
config = app_config['development']
config.APPLICATION_ROOT = '/rest/'
app.config.from_object(config)
else:
config = app_config['production']
config.APPLICATION_ROOT = '/rest/'
app.config.from_object(config)
with app.app_context():
register_converters(app)
register_error_pages(app)
register_blueprints(app)
return app
def create_rest_api(event_queue):
from cmdb.interface.config import app_config
from cmdb.utils.system_config import SystemConfigReader
system_config_reader = SystemConfigReader()
try:
cache_config = {
'DEBUG':
True,
'CACHE_TYPE':
system_config_reader.get_value('name', 'Cache'),
'CACHE_REDIS_HOST':
system_config_reader.get_value('host', 'Cache'),
'CACHE_REDIS_PORT':
system_config_reader.get_value('port', 'Cache'),
'CACHE_REDIS_PASSWORD':
system_config_reader.get_value('password', 'Cache'),
}
except (ImportError, CMDBError) as e:
LOGGER.debug(e.message)
cache_config = {'CACHE_TYPE': 'simple'}
from flask_caching import Cache
cache = Cache(config=cache_config)
# Create manager
from cmdb.data_storage.database_manager import DatabaseManagerMongo
app_database = DatabaseManagerMongo(
**system_config_reader.get_all_values_from_section('Database'))
object_manager = CmdbObjectManager(database_manager=app_database,
event_queue=event_queue)
log_manager = CmdbLogManager(database_manager=app_database)
security_manager = SecurityManager(database_manager=app_database)
user_manager = UserManager(database_manager=app_database)
exportd_job_manager = ExportdJobManagement(database_manager=app_database,
event_queue=event_queue)
exportd_log_manager = ExportdLogManager(database_manager=app_database)
media_file_manager = MediaFileManagement(database_manager=app_database)
docapi_tpl_manager = DocapiTemplateManager(database_manager=app_database)
# Create APP
from cmdb.interface.cmdb_app import BaseCmdbApp
app = BaseCmdbApp(__name__,
database_manager=app_database,
docapi_tpl_manager=docapi_tpl_manager,
media_file_manager=media_file_manager,
exportd_manager=exportd_job_manager,
exportd_log_manager=exportd_log_manager,
object_manager=object_manager,
log_manager=log_manager,
user_manager=user_manager,
security_manager=security_manager)
# Import App Extensions
from flask_cors import CORS
CORS(app)
import cmdb
cache.init_app(app)
cache.clear()
app.cache = cache
if cmdb.__MODE__ == 'DEBUG':
app.config.from_object(app_config['rest_development'])
LOGGER.info('RestAPI starting with config mode {}'.format(
app.config.get("ENV")))
elif cmdb.__MODE__ == 'TESTING':
app.config.from_object(app_config['testing'])
else:
app.config.from_object(app_config['rest'])
LOGGER.info('RestAPI starting with config mode {}'.format(
app.config.get("ENV")))
with app.app_context():
register_converters(app)
register_error_pages(app)
register_blueprints(app)
return app