def __init__(self, key_directory=None):
self.scr = SystemConfigReader()
ssc = SystemConfigReader()
database_options = ssc.get_all_values_from_section('Database')
self.__dbm = DatabaseManagerMongo(**database_options)
self.ssw = SystemSettingsWriter(self.__dbm)
self.key_directory = key_directory or SystemConfigReader.DEFAULT_CONFIG_LOCATION + "/keys"
def get_config_information(request_user: User):
ssc = SystemConfigReader()
config_dict = {'path': ssc.config_file, 'properties': []}
ssc = SystemConfigReader()
for section in ssc.get_sections():
section_values = []
for key, value in ssc.get_all_values_from_section(section).items():
section_values.append([key, value])
config_dict['properties'].append([section, section_values])
if len(config_dict) < 1:
return make_response(config_dict, 204)
return make_response(config_dict)
def __init__(self,
file,
file_type,
config: ObjectImporterConfig = None,
parser: BaseObjectParser = None,
object_manager: CmdbObjectManager = None,
request_user: UserModel = None):
"""
Basic importer super class for object imports
Normally should be started by start_import
Args:
file: File instance, name, content or loaded path to file
file_type: file type - used with content-type
config: importer configuration
parser: the parser instance based on content-type
object_manager: a instance of the object managers
request_user: the instance of the started user
"""
self.parser = parser
if object_manager:
self.object_manager = object_manager
else:
from cmdb.utils.system_config import SystemConfigReader
from cmdb.database.managers import DatabaseManagerMongo
object_manager = CmdbObjectManager(
database_manager=DatabaseManagerMongo(**SystemConfigReader(
).get_all_values_from_section('Database')))
self.object_manager = object_manager
self.request_user = request_user
super(ObjectImporter, self).__init__(file=file,
file_type=file_type,
config=config)
def __init__(self, message_queue, flag_shutdown, process_id=None):
"""Creates an instance of EventSenderAmqp
Args:
message_queue(queue.Queue): handler of a queue for sending events
flag_shutdown(threading.Event): flag for handling shutdown
process_id(str): process identifier (process name)
"""
super(EventSenderAmqp, self).__init__()
self.__queue = message_queue
self.__flag_shutdown = flag_shutdown
self.__process_id = process_id
# get configuration
self.__config_mq = SystemConfigReader().get_all_values_from_section(
'MessageQueueing')
self.__config_host = self.__config_mq.get("host", "127.0.0.1")
self.__config_port = self.__config_mq.get("port", "5672")
self.__config_username = self.__config_mq.get("username", "guest")
self.__config_password = self.__config_mq.get("password", "guest")
self.__config_exchange = self.__config_mq.get("exchange",
"datagerry.eventbus")
self.__config_retries = int(
self.__config_mq.get("connection_attempts", "5"))
self.__config_retrydelay = int(self.__config_mq.get(
"retry_delay", "6"))
self.__config_tls = False
if self.__config_mq.get("use_tls",
"False") in ("true", "True", "1", "yes"):
self.__config_tls = True
# define variables
self.__connection = None
self.__channel = None
def __init__(self, database_manager: DatabaseManagerMongo = None):
self.key_holder = KeyHolder()
self.header = {'alg': 'RS512'}
self.database_manager = database_manager or DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database'))
self.auth_module = AuthModule(
SystemSettingsReader(self.database_manager))
def _check_database():
"""
Checks whether the specified connection of the configuration is reachable.
Returns: Datebase if response
"""
from cmdb.database.errors.connection_errors import ServerTimeoutError
ssc = SystemConfigReader()
LOGGER.info(f'Checking database connection with {ssc.config_name} data')
database_options = ssc.get_all_values_from_section('Database')
dbm = DatabaseManagerMongo(**database_options)
try:
connection_test = dbm.connector.is_connected()
except ServerTimeoutError:
connection_test = False
LOGGER.debug(f'Database status is {connection_test}')
if connection_test is True:
return dbm
retries = 0
while retries < 3:
retries += 1
LOGGER.warning(
f'Retry {retries}: Checking database connection with {SystemConfigReader.DEFAULT_CONFIG_NAME} data'
)
connection_test = dbm.connector.is_connected()
if connection_test:
return dbm
return None
def authenticate(self, user_name: str, password: str, **kwargs) -> UserModel:
__dbm = DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database')
)
__user_manager = UserManager(__dbm)
try:
ldap_connection_status = self.connect()
LOGGER.debug(f'[LdapAuthenticationProvider] Connection status: {ldap_connection_status}')
except Exception as e:
LOGGER.error(f'[LdapAuthenticationProvider] Failed to connect to LDAP server - error: {e}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), e)
ldap_search_filter = self.config.search['searchfilter'].replace("%username%", user_name)
LOGGER.debug(f'[LdapAuthenticationProvider] Search Filter: {ldap_search_filter}')
search_result = self.__ldap_connection.search(self.config.search['basedn'], ldap_search_filter)
LOGGER.debug(f'[LdapAuthenticationProvider] Search result: {search_result}')
if not search_result or len(self.__ldap_connection.entries) == 0:
raise AuthenticationError(LdapAuthenticationProvider.get_name(), 'No matching entry')
for entry in self.__ldap_connection.entries:
LOGGER.debug(f'[LdapAuthenticationProvider] Entry: {entry}')
entry_dn = entry.entry_dn
try:
entry_connection_result = LdapAuthenticationProvider.Connection(self.__ldap_server, entry_dn, password,
auto_bind=True)
LOGGER.debug(f'[LdapAuthenticationProvider] UserModel connection result: {entry_connection_result}')
except Exception as e:
LOGGER.error(f'[LdapAuthenticationProvider] UserModel auth result: {e}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), e)
# Check if user exists
try:
user_instance: UserModel = __user_manager.get_by({'user_name': user_name})
except ManagerGetError as umge:
LOGGER.warning(f'[LdapAuthenticationProvider] UserModel exists on LDAP but not in database: {umge}')
LOGGER.debug(f'[LdapAuthenticationProvider] Try creating user: {user_name}')
try:
new_user_data = dict()
new_user_data['user_name'] = user_name
new_user_data['active'] = True
new_user_data['group_id'] = self.config.default_group
new_user_data['registration_time'] = datetime.now()
new_user_data['authenticator'] = LdapAuthenticationProvider.get_name()
except Exception as e:
LOGGER.debug(f'[LdapAuthenticationProvider] {e}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), e)
LOGGER.debug(f'[LdapAuthenticationProvider] New user was init')
try:
user_id = __user_manager.insert(new_user_data)
except ManagerInsertError as umie:
LOGGER.debug(f'[LdapAuthenticationProvider] {umie}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), umie)
try:
user_instance: UserModel = __user_manager.get(public_id=user_id)
except ManagerGetError as umge:
LOGGER.debug(f'[LdapAuthenticationProvider] {umge}')
raise AuthenticationError(LdapAuthenticationProvider.get_name(), umge)
return user_instance
def reload_config_reader(request_user: User):
ssc = SystemConfigReader()
ssc.setup()
LOGGER.warning('Reload config file!')
status = ssc.status()
if status:
current_app.cache.clear()
return make_response(status)
def _init_config_reader(config_file):
import os
path, filename = os.path.split(config_file)
if filename is not SystemConfigReader.DEFAULT_CONFIG_NAME or path is not SystemConfigReader.DEFAULT_CONFIG_LOCATION:
SystemConfigReader.RUNNING_CONFIG_NAME = filename
SystemConfigReader.RUNNING_CONFIG_LOCATION = path + '/'
SystemConfigReader(SystemConfigReader.RUNNING_CONFIG_NAME,
SystemConfigReader.RUNNING_CONFIG_LOCATION)
def __init__(self):
scr = SystemConfigReader()
self.database_manager = DatabaseManagerMongo(
**scr.get_all_values_from_section('Database'))
self.object_manager = CmdbObjectManager(
database_manager=self.database_manager)
self.log_manager = ExportdLogManager(
database_manager=self.database_manager)
super().__init__(self.database_manager)
def __init__(self, key_directory=None):
"""
Args:
key_directory: key based directory
"""
self.scr = SystemConfigReader()
self.__dbm = DatabaseManagerMongo(
**self.scr.get_all_values_from_section('Database'))
self.ssr = SystemSettingsReader(self.__dbm)
self.rsa_public = self.get_public_key()
self.rsa_private = self.get_private_key()
def __init__(self):
self.status = UpdateRoutine.UpateStatus.NOT
# check if settings are loaded
from cmdb.utils.system_config import SystemConfigReader
self.setup_system_config_reader = SystemConfigReader()
system_config_reader_status = self.setup_system_config_reader.status()
if system_config_reader_status is not True:
self.status = UpdateRoutine.UpateStatus.ERROR
raise RuntimeError(
f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \
system config reader status: {system_config_reader_status}')
def __init__(self, object_list: List[CmdbObject], request_user: UserModel, dt_render=False, ref_render=False,
object_manager: CmdbObjectManager = None):
self.object_list: List[CmdbObject] = object_list
self.request_user = request_user
self.dt_render = dt_render
self.ref_render = ref_render
from cmdb.utils.system_config import SystemConfigReader
database_manager = DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database')
)
self.object_manager = object_manager or CmdbObjectManager(database_manager=database_manager)
self.user_manager = UserManager(database_manager=database_manager)
def __init__(self, dbm: DatabaseManagerMongo):
self.status = SetupRoutine.SetupStatus.NOT
# check if settings are loaded
self.setup_system_config_reader = SystemConfigReader()
self.setup_database_manager = dbm
system_config_reader_status = self.setup_system_config_reader.status()
if system_config_reader_status is not True:
self.status = SetupRoutine.SetupStatus.ERROR
raise RuntimeError(
f'The system configuration files were loaded incorrectly or nothing has been loaded at all. - \
system config reader status: {system_config_reader_status}'
)
def __init__(self, job: ExportdJob):
self.job = job
self.exportvars = self.__get_exportvars()
self.destinations = self.__get__destinations()
scr = SystemConfigReader()
database_manager = DatabaseManagerMongo(
**scr.get_all_values_from_section('Database'))
self.__object_manager = CmdbObjectManager(
database_manager=database_manager)
self.log_manager = ExportdLogManager(database_manager=database_manager)
self.sources = self.__get_sources()
super(ExportdManagerBase, self).__init__(database_manager)
def _run(self):
# get queue for sending events
event_queue = self._event_manager.get_send_queue()
database_manager = DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database'))
# get WSGI app
app = DispatcherMiddleware(app=create_app(),
mounts={
'/docs':
create_docs_server(),
'/rest':
create_rest_api(database_manager,
event_queue)
})
# get gunicorn options
options = SystemConfigReader().get_all_values_from_section('WebServer')
# start gunicorn as own process
webserver = HTTPServer(app, options)
self.__webserver_proc = multiprocessing.Process(target=webserver.run)
self.__webserver_proc.start()
self.__webserver_proc.join()
def authenticate(self, user_name: str, password: str, **kwargs) -> UserModel:
__dbm = DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database')
)
__scm = SecurityManager(__dbm)
__user_manager = UserManager(__dbm)
LOGGER.info(f'[LocalAuthenticationProvider] Try login for user {user_name}')
try:
user: UserModel = __user_manager.get_by({'user_name': user_name})
except ManagerGetError as umge:
raise AuthenticationError(LocalAuthenticationProvider.get_name(), umge.message)
login_pass = __scm.generate_hmac(password)
if login_pass == user.password:
return user
raise AuthenticationError(LocalAuthenticationProvider.get_name(), 'UserModel not exists')
def __init__(self, event, state=False):
super(ExportdThread, self).__init__()
self.job = None
self.job_id = event.get_param("id")
self.type_id = event.get_param("type_id")
self.user_id = event.get_param("user_id")
self.event = event
self.is_active = state
self.exception_handling = None
scr = SystemConfigReader()
database_options = scr.get_all_values_from_section('Database')
self.__dbm = DatabaseManagerMongo(**database_options)
self.log_manager = ExportdLogManager(database_manager=self.__dbm)
self.exportd_job_manager = ExportdJobManagement(
database_manager=self.__dbm)
self.user_manager = UserManager(database_manager=self.__dbm)
def __init__(self,
receiver_callback,
flag_shutdown,
process_id=None,
event_types=["#"]):
"""Creates an instance of EventReceiverAmqp
Args:
receiver_callback(func): callback function for processing
received events
flag_shutdown(threading.Event): flag for handling shutdown
process_id(str): process identifier (process name)
event_types(list): list of event types, that will be processed
"""
super(EventReceiverAmqp, self).__init__()
self.__receiver_callback = receiver_callback
self.__flag_shutdown = flag_shutdown
self.__process_id = process_id
self.__event_types = event_types
# get configuration
self.__config_mq = SystemConfigReader().get_all_values_from_section(
'MessageQueueing')
self.__config_host = self.__config_mq.get("host", "127.0.0.1")
self.__config_port = self.__config_mq.get("port", "5672")
self.__config_username = self.__config_mq.get("username", "guest")
self.__config_password = self.__config_mq.get("password", "guest")
self.__config_exchange = self.__config_mq.get("exchange",
"datagerry.eventbus")
self.__config_retries = int(
self.__config_mq.get("connection_attempts", "5"))
self.__config_retrydelay = int(self.__config_mq.get(
"retry_delay", "6"))
self.__config_tls = False
if self.__config_mq.get("use_tls",
"False") in ("true", "True", "1", "yes"):
self.__config_tls = True
# define variables
self.__connection = None
self.__channel = None
def __init__(self, event: Event, state: bool = False):
scr = SystemConfigReader()
database_options = scr.get_all_values_from_section('Database')
database = DatabaseManagerMongo(**database_options)
super(ExportdThread, self).__init__()
self.job = None
self.job_id = int(event.get_param("id"))
self.type_id = int(
event.get_param("type_id")) if event.get_param("type_id") else None
self.user_id = int(event.get_param("user_id"))
self.event = event
self.is_active = state
self.exception_handling = None
self.object_manager = CmdbObjectManager(database_manager=database)
self.log_manager = ExportdLogManager(database_manager=database)
self.exportd_job_manager = ExportdJobManagement(
database_manager=database)
self.user_manager = UserManager(database_manager=database)
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import logging
import cmdb
from cmdb.interface.cmdb_app import BaseCmdbApp
from cmdb.interface.config import app_config
from cmdb.interface.docs.doc_routes import doc_pages
from cmdb.utils.error import CMDBError
LOGGER = logging.getLogger(__name__)
from cmdb.utils.system_config import SystemConfigReader
system_config_reader = SystemConfigReader()
def create_docs_server(event_queue):
# Create managers
from cmdb.data_storage.database_manager import DatabaseManagerMongo
app_database = DatabaseManagerMongo(
**system_config_reader.get_all_values_from_section('Database')
)
app = BaseCmdbApp(__name__, app_database)
if cmdb.__MODE__ == 'DEBUG':
config = app_config['development']
config.APPLICATION_ROOT = '/docs/'
app.config.from_object(config)
def test_config_less_instance(self):
from cmdb.utils.system_config import SystemConfigReader
system_config_reader = SystemConfigReader()
assert system_config_reader.config_file_less is True
from flask import abort, Response
from cmdb.data_storage.database_manager import DatabaseManagerMongo
from cmdb.framework.cmdb_errors import ObjectNotFoundError, TypeNotFoundError
from cmdb.file_export.export_types import ExportType
from cmdb.framework.cmdb_object_manager import CmdbObjectManager
try:
from cmdb.utils.error import CMDBError
except ImportError:
CMDBError = Exception
from cmdb.utils.system_config import SystemConfigReader
object_manager = CmdbObjectManager(database_manager=DatabaseManagerMongo(
**SystemConfigReader().get_all_values_from_section('Database')))
class FileExporter:
@staticmethod
def get_filetypes():
filetypes = [
"CsvExportType", "JsonExportType", "XlsxExportType",
"XmlExportType"
]
return filetypes
def __init__(self, object_type, export_type: ExportType, public_id: str,
**kwargs):
"""init of FileExporter
def main(args):
"""
Default application start function
Args:
args: start-options
"""
LOGGER.info("DATAGERRY starting...")
if args.debug:
_activate_debug()
_init_config_reader(args.config_file)
from cmdb.data_storage.database_connection import DatabaseConnectionError
# create / check connection database manager
dbm = None
try:
dbm = _check_database()
if not dbm:
raise DatabaseConnectionError('')
LOGGER.info("Database connection established.")
except CMDBError as conn_error:
LOGGER.critical(conn_error.message)
exit(1)
# check db-settings and run update if needed
if args.start:
from cmdb.__check__ import CheckRoutine
check_routine = CheckRoutine(dbm)
# check db-settings
try:
check_status = check_routine.checker()
except Exception as err:
LOGGER.error(err)
check_status = check_routine.get_check_status()
LOGGER.error(
f'The check did not go through as expected. Please run an update. \n Error: {err}'
)
if check_status == CheckRoutine.CheckStatus.HAS_UPDATES:
# run update
from cmdb.__update__ import UpdateRoutine
update_routine = UpdateRoutine()
try:
update_status = update_routine.start_update()
except RuntimeError as err:
LOGGER.error(err)
update_status = update_routine.get_updater_status()
LOGGER.warning(
f'The update did not go through as expected - Status {update_status}'
)
if update_status == UpdateRoutine.UpateStatus.FINISHED:
check_status = CheckRoutine.CheckStatus.FINISHED
else:
exit(1)
if check_status == CheckRoutine.CheckStatus.FINISHED:
# run setup if needed
from cmdb.__setup__ import SetupRoutine
setup_routine = SetupRoutine(dbm)
try:
setup_status = setup_routine.setup()
except RuntimeError as err:
LOGGER.error(err)
setup_status = setup_routine.get_setup_status()
LOGGER.warning(
f'The setup did not go through as expected - Status {setup_status}'
)
if setup_status == SetupRoutine.SetupStatus.FINISHED:
pass
else:
exit(1)
else:
pass
if args.keys:
from cmdb.__setup__ import SetupRoutine
setup_routine = SetupRoutine(dbm)
setup_status = None
try:
setup_routine.init_keys()
except RuntimeError as err:
LOGGER.error(err)
setup_status = setup_routine.get_setup_status()
LOGGER.warning(
f'The key generation did not go through as expected - Status {setup_status}'
)
if setup_status == SetupRoutine.SetupStatus.FINISHED:
exit(0)
else:
exit(1)
if args.test_data:
_activate_debug()
from cmdb.utils.data_factory import DataFactory
ssc = SystemConfigReader()
database_options = ssc.get_all_values_from_section('Database')
dbm = DatabaseManagerMongo(**database_options)
db_name = dbm.get_database_name()
LOGGER.warning(f'Inserting test-data into: {db_name}')
try:
factory = DataFactory(database_manager=dbm)
ack = factory.insert_data()
LOGGER.warning("Test-data was successfully added".format(
dbm.get_database_name()))
if len(ack) > 0:
LOGGER.critical(
"Error while inserting test-data: {} - dropping database".
format(ack))
dbm.drop(db_name) # cleanup database
except (Exception, CMDBError) as e:
import traceback
traceback.print_tb(e.__traceback__)
dbm.drop(db_name) # cleanup database
exit(1)
if args.start:
_start_app()
sleep(0.2) # prevent logger output
LOGGER.info("DATAGERRY successfully started")
def create_rest_api(event_queue):
from cmdb.interface.config import app_config
from cmdb.utils.system_config import SystemConfigReader
system_config_reader = SystemConfigReader()
try:
cache_config = {
'DEBUG':
True,
'CACHE_TYPE':
system_config_reader.get_value('name', 'Cache'),
'CACHE_REDIS_HOST':
system_config_reader.get_value('host', 'Cache'),
'CACHE_REDIS_PORT':
system_config_reader.get_value('port', 'Cache'),
'CACHE_REDIS_PASSWORD':
system_config_reader.get_value('password', 'Cache'),
}
except (ImportError, CMDBError) as e:
LOGGER.debug(e.message)
cache_config = {'CACHE_TYPE': 'simple'}
from flask_caching import Cache
cache = Cache(config=cache_config)
# Create manager
from cmdb.data_storage.database_manager import DatabaseManagerMongo
app_database = DatabaseManagerMongo(
**system_config_reader.get_all_values_from_section('Database'))
object_manager = CmdbObjectManager(database_manager=app_database,
event_queue=event_queue)
log_manager = CmdbLogManager(database_manager=app_database)
security_manager = SecurityManager(database_manager=app_database)
user_manager = UserManager(database_manager=app_database)
exportd_job_manager = ExportdJobManagement(database_manager=app_database,
event_queue=event_queue)
exportd_log_manager = ExportdLogManager(database_manager=app_database)
media_file_manager = MediaFileManagement(database_manager=app_database)
docapi_tpl_manager = DocapiTemplateManager(database_manager=app_database)
# Create APP
from cmdb.interface.cmdb_app import BaseCmdbApp
app = BaseCmdbApp(__name__,
database_manager=app_database,
docapi_tpl_manager=docapi_tpl_manager,
media_file_manager=media_file_manager,
exportd_manager=exportd_job_manager,
exportd_log_manager=exportd_log_manager,
object_manager=object_manager,
log_manager=log_manager,
user_manager=user_manager,
security_manager=security_manager)
# Import App Extensions
from flask_cors import CORS
CORS(app)
import cmdb
cache.init_app(app)
cache.clear()
app.cache = cache
if cmdb.__MODE__ == 'DEBUG':
app.config.from_object(app_config['rest_development'])
LOGGER.info('RestAPI starting with config mode {}'.format(
app.config.get("ENV")))
elif cmdb.__MODE__ == 'TESTING':
app.config.from_object(app_config['testing'])
else:
app.config.from_object(app_config['rest'])
LOGGER.info('RestAPI starting with config mode {}'.format(
app.config.get("ENV")))
with app.app_context():
register_converters(app)
register_error_pages(app)
register_blueprints(app)
return app
def create_rest_api(event_queue):
from cmdb.interface.config import app_config
from cmdb.utils.system_config import SystemConfigReader
system_config_reader = SystemConfigReader()
# Create managers
from cmdb.data_storage.database_manager import DatabaseManagerMongo
app_database = DatabaseManagerMongo(
**system_config_reader.get_all_values_from_section('Database'))
object_manager = CmdbObjectManager(database_manager=app_database,
event_queue=event_queue)
log_manager = CmdbLogManager(database_manager=app_database)
security_manager = SecurityManager(database_manager=app_database)
user_manager = UserManager(database_manager=app_database)
exportd_job_manager = ExportdJobManagement(database_manager=app_database,
event_queue=event_queue)
exportd_log_manager = ExportdLogManager(database_manager=app_database)
media_file_manager = MediaFileManagement(database_manager=app_database)
docapi_tpl_manager = DocapiTemplateManager(database_manager=app_database)
# Create APP
from cmdb.interface.cmdb_app import BaseCmdbApp
app = BaseCmdbApp(__name__,
database_manager=app_database,
docapi_tpl_manager=docapi_tpl_manager,
media_file_manager=media_file_manager,
exportd_manager=exportd_job_manager,
exportd_log_manager=exportd_log_manager,
object_manager=object_manager,
log_manager=log_manager,
user_manager=user_manager,
security_manager=security_manager)
app.url_map.strict_slashes = True
# Import App Extensions
from flask_cors import CORS
CORS(app, expose_headers=['X-API-Version', 'X-Total-Count'])
import cmdb
if cmdb.__MODE__ == 'DEBUG':
config = app_config['development']
config.APPLICATION_ROOT = '/rest/'
app.config.from_object(config)
else:
config = app_config['production']
config.APPLICATION_ROOT = '/rest/'
app.config.from_object(config)
with app.app_context():
register_converters(app)
register_error_pages(app)
register_blueprints(app)
return app