def get_database_config(parsed: ConfigParser, manager: ConfigManager) -> dict: """ Generate a populated configuration dictionary. TODO: This should be shared with dbManager.py. :param parsed: ConfigParser :param manager: ConfigManager :return: Dictionary """ config: dict = {} config['engine']: str = parsed.get('database', 'engine') if \ parsed.has_option('database', 'engine') else \ 'sqlite' config['host']: str = parsed.get('database', 'host') if \ parsed.has_option('database', 'host') else \ 'localhost' config['username']: str = parsed.get('database', 'username') if \ parsed.has_option('database', 'username') else \ manager.getDbUser() config['password']: str = parsed.get('database', 'password') if \ parsed.has_option('database', 'password') else \ manager.getDbPassword() if config['engine'] == 'sqlite': config['port']: Optional[int] = None config['path']: str = parsed.get('database', 'path') if \ parsed.has_option('database', 'path') else \ os.path.join( manager.getEtcDir(), manager.getDbSchema() + '.sqlite' ) elif config['engine'] == 'mysql': config['port']: Optional[int] = manager.get('database', 'port') if \ manager.has_option('database', 'port') else \ 3306 config['path']: str = manager.getDbSchema() else: raise NotImplementedError('{} is not supported'.format( config['engine'])) return config
class DbManagerBase(TortugaObjectManager): """ Class for db management. :param engine: a SQLAlchemy database engine instance :param init: a flag that is set when the database has not yet been initialized. If this flag is set, not attempts will be made to load/map kit tables. This flag is cleared once the database has been initialized. """ def __init__(self, engine=None): super(DbManagerBase, self).__init__() self.Session = None if not engine: self._cm = ConfigManager() self._dbConfig = self._refreshDbConfig() engineURI = self.__getDbEngineURI() if self._dbConfig['engine'] == 'mysql': # Set default SQLAlchemy engine arguments for MySQL kwargs = { 'pool_size': 10, 'max_overflow': 2, 'pool_recycle': 600, 'echo': False, 'pool_timeout': 60, } else: kwargs = {} self._engine = sqlalchemy.create_engine(engineURI, **kwargs) else: self._engine = engine self._metadata = sqlalchemy.MetaData(self._engine) self._mapped_tables = {} def _map_db_tables(self): # # Make sure all kit table mappers have been registered # load_kits() for kit_installer_class in get_all_kit_installers(): kit_installer = kit_installer_class() kit_installer.register_database_table_mappers() # # Map all tables that haven't yet been mapped # for table_mapper in get_all_table_mappers(): key = table_mapper.__name__ if key not in self._mapped_tables.keys(): logger.debug('Mapping table: {}'.format(key)) self._mapped_tables[key] = table_mapper() self._mapped_tables[key].map(self) @property def engine(self): """ SQLAlchemy Engine object property """ self._map_db_tables() return self._engine def session(self): """ Database session context manager """ return SessionContextManager(self) def init_database(self): # # Create tables # self._map_db_tables() try: self._metadata.create_all(self.engine) except Exception: self.getLogger().exception('SQLAlchemy raised exception') raise DbError('Check database settings or credentials') @property def metadata(self): return self._metadata def __getDbEngineURI(self): dbPort = self._dbConfig['port'] dbHost = self._dbConfig['host'] engine = self._dbConfig['engine'] dbUser = self._dbConfig['username'] dbPassword = self._dbConfig['password'] if engine == 'sqlite': engineURI = 'sqlite:///%s' % (self._dbConfig['path']) else: if dbUser is not None: if dbPassword is not None: userspec = '%s:%s' % (dbUser, dbPassword) else: userspec = dbUser else: userspec = None if dbPort is not None: hostspec = '%s:%s' % (dbHost, dbPort) else: hostspec = dbHost if userspec is not None: engineURI = '%s://%s@%s/%s' % (engine, userspec, hostspec, self._cm.getDbSchema()) else: engineURI = '%s://%s/%s' % (engine, hostspec, self._cm.getDbSchema()) return engineURI def _getDefaultDbEngine(self): \ # pylint: disable=no-self-use return 'sqlite' def _getDefaultDbHost(self): \ # pylint: disable=no-self-use return 'localhost' def _getDefaultDbPort(self, engine): \ # pylint: disable=no-self-use # MySQL default port if engine == 'mysql': return 3306 return None def _getDefaultDbUserName(self): return self._cm.getDbUser() def _getDefaultDbPassword(self): if os.path.exists(self._cm.getDbPasswordFile()): with open(self._cm.getDbPasswordFile()) as fp: dbPassword = fp.read() else: dbPassword = None return dbPassword def _refreshDbConfig(self, cfg=None): dbConfig = {} if cfg is None: cfg = configparser.ConfigParser() cfg.read(os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini')) # Database engine val = cfg.get('database', 'engine').strip().lower() \ if cfg.has_option('database', 'engine') else \ self._getDefaultDbEngine() dbConfig['engine'] = val if dbConfig['engine'] == 'sqlite': # If database is sqlite, read the path dbConfig['path'] = cfg.get('database', 'path') \ if cfg.has_section('database') and \ cfg.has_option('database', 'path') else \ os.path.join(self._cm.getEtcDir(), self._cm.getDbSchema() + '.sqlite') # Database host val = cfg.get('database', 'host') \ if cfg.has_option('database', 'host') else \ self._getDefaultDbHost() dbConfig['host'] = val # Database port val = cfg.get('database', 'port') \ if cfg.has_option('database', 'port') else None dbConfig['port'] = val if val else self._getDefaultDbPort( engine=dbConfig['engine']) # Database username val = cfg.get('database', 'username') \ if cfg.has_option('database', 'username') \ else self._getDefaultDbUserName() dbConfig['username'] = val # Database password val = cfg.get('database', 'password') \ if cfg.has_option('database', 'password') \ else self._getDefaultDbPassword() dbConfig['password'] = val return dbConfig def get_backend_opts(self): \ # pylint: disable=no-self-use return { 'mysql_engine': 'InnoDB', } def getMetadataTable(self, table): return self._metadata.tables[table] def openSession(self): """ Open db session. """ session_factory = sqlalchemy.orm.sessionmaker(bind=self.engine) self.Session = sqlalchemy.orm.scoped_session(session_factory) return self.Session() def closeSession(self): """Close scoped_session.""" self.Session.remove()
class DbManager(TortugaObjectManager): """ Class for db management. :param engine: a SQLAlchemy database engine instance :param init: a flag that is set when the database has not yet been initialized. If this flag is set, not attempts will be made to load/map kit tables. This flag is cleared once the database has been initialized. """ def __init__(self, engine=None): super().__init__() if not engine: self._cm = ConfigManager() self._dbConfig = self._refreshDbConfig() engineURI = self.__getDbEngineURI() if self._dbConfig['engine'] == 'sqlite' and \ not os.path.exists(self._dbConfig['path']): # Ensure SQLite database file is created with proper permissions fd = os.open( self._dbConfig['path'], os.O_CREAT, mode=0o600) os.close(fd) self._engine = sqlalchemy.create_engine(engineURI) else: self._engine = engine self.Session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=self.engine)) def _register_database_tables(self): for kit_installer_class in get_all_kit_installers(): kit_installer = kit_installer_class() kit_installer.register_database_tables() @property def engine(self): """ SQLAlchemy Engine object property """ self._register_database_tables() return self._engine def session(self): """ Database session context manager """ return SessionContextManager(self) def init_database(self): # # Create tables # self._register_database_tables() try: ModelBase.metadata.create_all(self.engine) except Exception: self._logger.exception('SQLAlchemy raised exception') raise DbError('Check database settings or credentials') @property def metadata(self): return self._metadata def __getDbEngineURI(self): dbPort = self._dbConfig['port'] dbHost = self._dbConfig['host'] engine = self._dbConfig['engine'] dbUser = self._dbConfig['username'] dbPassword = self._dbConfig['password'] if engine == 'sqlite': engineURI = 'sqlite:///%s' % (self._dbConfig['path']) else: if dbUser is not None: if dbPassword is not None: userspec = '%s:%s' % (dbUser, dbPassword) else: userspec = dbUser else: userspec = None if dbPort is not None: hostspec = '%s:%s' % (dbHost, dbPort) else: hostspec = dbHost engineURI = f'{engine}+pymysql' if engine == 'mysql' else engine engineURI += '://' if userspec is not None: engineURI += f'{userspec}@' engineURI += f'{hostspec}' + '/{}'.format(self._cm.getDbSchema()) return engineURI def _getDefaultDbEngine(self): \ # pylint: disable=no-self-use return 'sqlite' def _getDefaultDbHost(self): \ # pylint: disable=no-self-use return 'localhost' def _getDefaultDbPort(self, engine): \ # pylint: disable=no-self-use # MySQL default port if engine == 'mysql': return 3306 return None def _getDefaultDbUserName(self): return self._cm.getDbUser() def _getDefaultDbPassword(self): if os.path.exists(self._cm.getDbPasswordFile()): with open(self._cm.getDbPasswordFile()) as fp: dbPassword = fp.read() else: dbPassword = None return dbPassword def _refreshDbConfig(self, cfg=None): dbConfig = {} if cfg is None: cfg = configparser.ConfigParser() cfg.read(os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini')) # Database engine val = cfg.get('database', 'engine').strip().lower() \ if cfg.has_option('database', 'engine') else \ self._getDefaultDbEngine() dbConfig['engine'] = val if dbConfig['engine'] == 'sqlite': # If database is sqlite, read the path dbConfig['path'] = cfg.get('database', 'path') \ if cfg.has_section('database') and \ cfg.has_option('database', 'path') else \ os.path.join(self._cm.getEtcDir(), self._cm.getDbSchema() + '.sqlite') # Database host val = cfg.get('database', 'host') \ if cfg.has_option('database', 'host') else \ self._getDefaultDbHost() dbConfig['host'] = val # Database port val = cfg.get('database', 'port') \ if cfg.has_option('database', 'port') else None dbConfig['port'] = val if val else self._getDefaultDbPort( engine=dbConfig['engine']) # Database username val = cfg.get('database', 'username') \ if cfg.has_option('database', 'username') \ else self._getDefaultDbUserName() dbConfig['username'] = val # Database password val = cfg.get('database', 'password') \ if cfg.has_option('database', 'password') \ else self._getDefaultDbPassword() dbConfig['password'] = val return dbConfig def get_backend_opts(self): \ # pylint: disable=no-self-use return { 'mysql_engine': 'InnoDB', } def getMetadataTable(self, table): return self._metadata.tables[table] def openSession(self): """ Open db session. """ return self.Session() def closeSession(self): """Close scoped_session.""" self.Session.remove()
class DbManager(TortugaObjectManager): """ Class for db management. :param engine: a SQLAlchemy database engine instance :param init: a flag that is set when the database has not yet been initialized. If this flag is set, not attempts will be made to load/map kit tables. This flag is cleared once the database has been initialized. """ def __init__(self, engine=None): super().__init__() if not engine: self._cm = ConfigManager() engine = self._cm.getDbEngine() schema = self._cm.getDbSchema() engineURI = self.__getDbEngineURI() if engine == 'sqlite' and not os.path.exists(schema): # Ensure SQLite database file is created with proper # permissions fd = os.open(schema, os.O_CREAT, mode=0o600) os.close(fd) self._engine = sqlalchemy.create_engine(engineURI, pool_size=100, max_overflow=100, pool_pre_ping=True) else: self._engine = engine self.Session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=self.engine)) def _register_database_tables(self): for kit_installer_class in get_all_kit_installers(): kit_installer = kit_installer_class() kit_installer.register_database_tables() @property def engine(self): """ SQLAlchemy Engine object property """ self._register_database_tables() return self._engine def session(self): """ Database session context manager """ return SessionContextManager(self) def init_database(self): # # Create tables # self._register_database_tables() ModelBase.metadata.create_all(self.engine) @property def metadata(self): return self._metadata def __getDbEngineURI(self): engine = self._cm.getDbEngine() schema = self._cm.getDbSchema() driver = '' host = '' port = '' user = '' password = '' if engine == 'mysql': driver = "+pymysql" host = self._cm.getDbHost() port = self._cm.getDbPort() user = self._cm.getDbUser() password = self._cm.getDbPassword() userspec = '' if user: if password: userspec = '{}:{}@'.format(user, password) else: userspec = '{}@'.format(user) hostspec = '' if host: if port: hostspec = '{}:{}'.format(host, port) else: hostspec = host engineURI = "{}{}://{}{}/{}".format(engine, driver, userspec, hostspec, schema) return engineURI def getMetadataTable(self, table): return self._metadata.tables[table] def openSession(self): """ Open db session. """ return self.Session() def closeSession(self): """Close scoped_session.""" self.Session.remove()