def get_database_config(parsed: ConfigParser, manager: ConfigManager) -> dict:
"""
Generate a populated
configuration dictionary.
TODO: This should be shared
with dbManager.py.
:param parsed: ConfigParser
:param manager: ConfigManager
:return: Dictionary
"""
config: dict = {}
config['engine']: str = parsed.get('database', 'engine') if \
parsed.has_option('database', 'engine') else \
'sqlite'
config['host']: str = parsed.get('database', 'host') if \
parsed.has_option('database', 'host') else \
'localhost'
config['username']: str = parsed.get('database', 'username') if \
parsed.has_option('database', 'username') else \
manager.getDbUser()
config['password']: str = parsed.get('database', 'password') if \
parsed.has_option('database', 'password') else \
manager.getDbPassword()
if config['engine'] == 'sqlite':
config['port']: Optional[int] = None
config['path']: str = parsed.get('database', 'path') if \
parsed.has_option('database', 'path') else \
os.path.join(
manager.getEtcDir(),
manager.getDbSchema() + '.sqlite'
)
elif config['engine'] == 'mysql':
config['port']: Optional[int] = manager.get('database', 'port') if \
manager.has_option('database', 'port') else \
3306
config['path']: str = manager.getDbSchema()
else:
raise NotImplementedError('{} is not supported'.format(
config['engine']))
return config
class DbManagerBase(TortugaObjectManager):
"""
Class for db management.
:param engine: a SQLAlchemy database engine instance
:param init: a flag that is set when the database has not yet
been initialized. If this flag is set, not attempts
will be made to load/map kit tables. This flag is
cleared once the database has been initialized.
"""
def __init__(self, engine=None):
super(DbManagerBase, self).__init__()
self.Session = None
if not engine:
self._cm = ConfigManager()
self._dbConfig = self._refreshDbConfig()
engineURI = self.__getDbEngineURI()
if self._dbConfig['engine'] == 'mysql':
# Set default SQLAlchemy engine arguments for MySQL
kwargs = {
'pool_size': 10,
'max_overflow': 2,
'pool_recycle': 600,
'echo': False,
'pool_timeout': 60,
}
else:
kwargs = {}
self._engine = sqlalchemy.create_engine(engineURI, **kwargs)
else:
self._engine = engine
self._metadata = sqlalchemy.MetaData(self._engine)
self._mapped_tables = {}
def _map_db_tables(self):
#
# Make sure all kit table mappers have been registered
#
load_kits()
for kit_installer_class in get_all_kit_installers():
kit_installer = kit_installer_class()
kit_installer.register_database_table_mappers()
#
# Map all tables that haven't yet been mapped
#
for table_mapper in get_all_table_mappers():
key = table_mapper.__name__
if key not in self._mapped_tables.keys():
logger.debug('Mapping table: {}'.format(key))
self._mapped_tables[key] = table_mapper()
self._mapped_tables[key].map(self)
@property
def engine(self):
"""
SQLAlchemy Engine object property
"""
self._map_db_tables()
return self._engine
def session(self):
"""
Database session context manager
"""
return SessionContextManager(self)
def init_database(self):
#
# Create tables
#
self._map_db_tables()
try:
self._metadata.create_all(self.engine)
except Exception:
self.getLogger().exception('SQLAlchemy raised exception')
raise DbError('Check database settings or credentials')
@property
def metadata(self):
return self._metadata
def __getDbEngineURI(self):
dbPort = self._dbConfig['port']
dbHost = self._dbConfig['host']
engine = self._dbConfig['engine']
dbUser = self._dbConfig['username']
dbPassword = self._dbConfig['password']
if engine == 'sqlite':
engineURI = 'sqlite:///%s' % (self._dbConfig['path'])
else:
if dbUser is not None:
if dbPassword is not None:
userspec = '%s:%s' % (dbUser, dbPassword)
else:
userspec = dbUser
else:
userspec = None
if dbPort is not None:
hostspec = '%s:%s' % (dbHost, dbPort)
else:
hostspec = dbHost
if userspec is not None:
engineURI = '%s://%s@%s/%s' % (engine, userspec, hostspec,
self._cm.getDbSchema())
else:
engineURI = '%s://%s/%s' % (engine, hostspec,
self._cm.getDbSchema())
return engineURI
def _getDefaultDbEngine(self): \
# pylint: disable=no-self-use
return 'sqlite'
def _getDefaultDbHost(self): \
# pylint: disable=no-self-use
return 'localhost'
def _getDefaultDbPort(self, engine): \
# pylint: disable=no-self-use
# MySQL default port
if engine == 'mysql':
return 3306
return None
def _getDefaultDbUserName(self):
return self._cm.getDbUser()
def _getDefaultDbPassword(self):
if os.path.exists(self._cm.getDbPasswordFile()):
with open(self._cm.getDbPasswordFile()) as fp:
dbPassword = fp.read()
else:
dbPassword = None
return dbPassword
def _refreshDbConfig(self, cfg=None):
dbConfig = {}
if cfg is None:
cfg = configparser.ConfigParser()
cfg.read(os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini'))
# Database engine
val = cfg.get('database', 'engine').strip().lower() \
if cfg.has_option('database', 'engine') else \
self._getDefaultDbEngine()
dbConfig['engine'] = val
if dbConfig['engine'] == 'sqlite':
# If database is sqlite, read the path
dbConfig['path'] = cfg.get('database', 'path') \
if cfg.has_section('database') and \
cfg.has_option('database', 'path') else \
os.path.join(self._cm.getEtcDir(),
self._cm.getDbSchema() + '.sqlite')
# Database host
val = cfg.get('database', 'host') \
if cfg.has_option('database', 'host') else \
self._getDefaultDbHost()
dbConfig['host'] = val
# Database port
val = cfg.get('database', 'port') \
if cfg.has_option('database', 'port') else None
dbConfig['port'] = val if val else self._getDefaultDbPort(
engine=dbConfig['engine'])
# Database username
val = cfg.get('database', 'username') \
if cfg.has_option('database', 'username') \
else self._getDefaultDbUserName()
dbConfig['username'] = val
# Database password
val = cfg.get('database', 'password') \
if cfg.has_option('database', 'password') \
else self._getDefaultDbPassword()
dbConfig['password'] = val
return dbConfig
def get_backend_opts(self): \
# pylint: disable=no-self-use
return {
'mysql_engine': 'InnoDB',
}
def getMetadataTable(self, table):
return self._metadata.tables[table]
def openSession(self):
""" Open db session. """
session_factory = sqlalchemy.orm.sessionmaker(bind=self.engine)
self.Session = sqlalchemy.orm.scoped_session(session_factory)
return self.Session()
def closeSession(self):
"""Close scoped_session."""
self.Session.remove()
class DbManager(TortugaObjectManager):
"""
Class for db management.
:param engine: a SQLAlchemy database engine instance
:param init: a flag that is set when the database has not yet
been initialized. If this flag is set, not attempts
will be made to load/map kit tables. This flag is
cleared once the database has been initialized.
"""
def __init__(self, engine=None):
super().__init__()
if not engine:
self._cm = ConfigManager()
self._dbConfig = self._refreshDbConfig()
engineURI = self.__getDbEngineURI()
if self._dbConfig['engine'] == 'sqlite' and \
not os.path.exists(self._dbConfig['path']):
# Ensure SQLite database file is created with proper permissions
fd = os.open(
self._dbConfig['path'], os.O_CREAT, mode=0o600)
os.close(fd)
self._engine = sqlalchemy.create_engine(engineURI)
else:
self._engine = engine
self.Session = sqlalchemy.orm.scoped_session(
sqlalchemy.orm.sessionmaker(bind=self.engine))
def _register_database_tables(self):
for kit_installer_class in get_all_kit_installers():
kit_installer = kit_installer_class()
kit_installer.register_database_tables()
@property
def engine(self):
"""
SQLAlchemy Engine object property
"""
self._register_database_tables()
return self._engine
def session(self):
"""
Database session context manager
"""
return SessionContextManager(self)
def init_database(self):
#
# Create tables
#
self._register_database_tables()
try:
ModelBase.metadata.create_all(self.engine)
except Exception:
self._logger.exception('SQLAlchemy raised exception')
raise DbError('Check database settings or credentials')
@property
def metadata(self):
return self._metadata
def __getDbEngineURI(self):
dbPort = self._dbConfig['port']
dbHost = self._dbConfig['host']
engine = self._dbConfig['engine']
dbUser = self._dbConfig['username']
dbPassword = self._dbConfig['password']
if engine == 'sqlite':
engineURI = 'sqlite:///%s' % (self._dbConfig['path'])
else:
if dbUser is not None:
if dbPassword is not None:
userspec = '%s:%s' % (dbUser, dbPassword)
else:
userspec = dbUser
else:
userspec = None
if dbPort is not None:
hostspec = '%s:%s' % (dbHost, dbPort)
else:
hostspec = dbHost
engineURI = f'{engine}+pymysql' if engine == 'mysql' else engine
engineURI += '://'
if userspec is not None:
engineURI += f'{userspec}@'
engineURI += f'{hostspec}' + '/{}'.format(self._cm.getDbSchema())
return engineURI
def _getDefaultDbEngine(self): \
# pylint: disable=no-self-use
return 'sqlite'
def _getDefaultDbHost(self): \
# pylint: disable=no-self-use
return 'localhost'
def _getDefaultDbPort(self, engine): \
# pylint: disable=no-self-use
# MySQL default port
if engine == 'mysql':
return 3306
return None
def _getDefaultDbUserName(self):
return self._cm.getDbUser()
def _getDefaultDbPassword(self):
if os.path.exists(self._cm.getDbPasswordFile()):
with open(self._cm.getDbPasswordFile()) as fp:
dbPassword = fp.read()
else:
dbPassword = None
return dbPassword
def _refreshDbConfig(self, cfg=None):
dbConfig = {}
if cfg is None:
cfg = configparser.ConfigParser()
cfg.read(os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini'))
# Database engine
val = cfg.get('database', 'engine').strip().lower() \
if cfg.has_option('database', 'engine') else \
self._getDefaultDbEngine()
dbConfig['engine'] = val
if dbConfig['engine'] == 'sqlite':
# If database is sqlite, read the path
dbConfig['path'] = cfg.get('database', 'path') \
if cfg.has_section('database') and \
cfg.has_option('database', 'path') else \
os.path.join(self._cm.getEtcDir(),
self._cm.getDbSchema() + '.sqlite')
# Database host
val = cfg.get('database', 'host') \
if cfg.has_option('database', 'host') else \
self._getDefaultDbHost()
dbConfig['host'] = val
# Database port
val = cfg.get('database', 'port') \
if cfg.has_option('database', 'port') else None
dbConfig['port'] = val if val else self._getDefaultDbPort(
engine=dbConfig['engine'])
# Database username
val = cfg.get('database', 'username') \
if cfg.has_option('database', 'username') \
else self._getDefaultDbUserName()
dbConfig['username'] = val
# Database password
val = cfg.get('database', 'password') \
if cfg.has_option('database', 'password') \
else self._getDefaultDbPassword()
dbConfig['password'] = val
return dbConfig
def get_backend_opts(self): \
# pylint: disable=no-self-use
return {
'mysql_engine': 'InnoDB',
}
def getMetadataTable(self, table):
return self._metadata.tables[table]
def openSession(self):
""" Open db session. """
return self.Session()
def closeSession(self):
"""Close scoped_session."""
self.Session.remove()
class DbManager(TortugaObjectManager):
"""
Class for db management.
:param engine: a SQLAlchemy database engine instance
:param init: a flag that is set when the database has not yet
been initialized. If this flag is set, not attempts
will be made to load/map kit tables. This flag is
cleared once the database has been initialized.
"""
def __init__(self, engine=None):
super().__init__()
if not engine:
self._cm = ConfigManager()
engine = self._cm.getDbEngine()
schema = self._cm.getDbSchema()
engineURI = self.__getDbEngineURI()
if engine == 'sqlite' and not os.path.exists(schema):
# Ensure SQLite database file is created with proper
# permissions
fd = os.open(schema, os.O_CREAT, mode=0o600)
os.close(fd)
self._engine = sqlalchemy.create_engine(engineURI,
pool_size=100,
max_overflow=100,
pool_pre_ping=True)
else:
self._engine = engine
self.Session = sqlalchemy.orm.scoped_session(
sqlalchemy.orm.sessionmaker(bind=self.engine))
def _register_database_tables(self):
for kit_installer_class in get_all_kit_installers():
kit_installer = kit_installer_class()
kit_installer.register_database_tables()
@property
def engine(self):
"""
SQLAlchemy Engine object property
"""
self._register_database_tables()
return self._engine
def session(self):
"""
Database session context manager
"""
return SessionContextManager(self)
def init_database(self):
#
# Create tables
#
self._register_database_tables()
ModelBase.metadata.create_all(self.engine)
@property
def metadata(self):
return self._metadata
def __getDbEngineURI(self):
engine = self._cm.getDbEngine()
schema = self._cm.getDbSchema()
driver = ''
host = ''
port = ''
user = ''
password = ''
if engine == 'mysql':
driver = "+pymysql"
host = self._cm.getDbHost()
port = self._cm.getDbPort()
user = self._cm.getDbUser()
password = self._cm.getDbPassword()
userspec = ''
if user:
if password:
userspec = '{}:{}@'.format(user, password)
else:
userspec = '{}@'.format(user)
hostspec = ''
if host:
if port:
hostspec = '{}:{}'.format(host, port)
else:
hostspec = host
engineURI = "{}{}://{}{}/{}".format(engine, driver, userspec, hostspec,
schema)
return engineURI
def getMetadataTable(self, table):
return self._metadata.tables[table]
def openSession(self):
""" Open db session. """
return self.Session()
def closeSession(self):
"""Close scoped_session."""
self.Session.remove()
