def get_database_config(parsed: ConfigParser, manager: ConfigManager) -> dict: """ Generate a populated configuration dictionary. TODO: This should be shared with dbManager.py. :param parsed: ConfigParser :param manager: ConfigManager :return: Dictionary """ config: dict = {} config['engine']: str = parsed.get('database', 'engine') if \ parsed.has_option('database', 'engine') else \ 'sqlite' config['host']: str = parsed.get('database', 'host') if \ parsed.has_option('database', 'host') else \ 'localhost' config['username']: str = parsed.get('database', 'username') if \ parsed.has_option('database', 'username') else \ manager.getDbUser() config['password']: str = parsed.get('database', 'password') if \ parsed.has_option('database', 'password') else \ manager.getDbPassword() if config['engine'] == 'sqlite': config['port']: Optional[int] = None config['path']: str = parsed.get('database', 'path') if \ parsed.has_option('database', 'path') else \ os.path.join( manager.getEtcDir(), manager.getDbSchema() + '.sqlite' ) elif config['engine'] == 'mysql': config['port']: Optional[int] = manager.get('database', 'port') if \ manager.has_option('database', 'port') else \ 3306 config['path']: str = manager.getDbSchema() else: raise NotImplementedError('{} is not supported'.format( config['engine'])) return config
class DbManagerBase(TortugaObjectManager): """ Class for db management. :param engine: a SQLAlchemy database engine instance :param init: a flag that is set when the database has not yet been initialized. If this flag is set, not attempts will be made to load/map kit tables. This flag is cleared once the database has been initialized. """ def __init__(self, engine=None): super(DbManagerBase, self).__init__() self.Session = None if not engine: self._cm = ConfigManager() self._dbConfig = self._refreshDbConfig() engineURI = self.__getDbEngineURI() if self._dbConfig['engine'] == 'mysql': # Set default SQLAlchemy engine arguments for MySQL kwargs = { 'pool_size': 10, 'max_overflow': 2, 'pool_recycle': 600, 'echo': False, 'pool_timeout': 60, } else: kwargs = {} self._engine = sqlalchemy.create_engine(engineURI, **kwargs) else: self._engine = engine self._metadata = sqlalchemy.MetaData(self._engine) self._mapped_tables = {} def _map_db_tables(self): # # Make sure all kit table mappers have been registered # load_kits() for kit_installer_class in get_all_kit_installers(): kit_installer = kit_installer_class() kit_installer.register_database_table_mappers() # # Map all tables that haven't yet been mapped # for table_mapper in get_all_table_mappers(): key = table_mapper.__name__ if key not in self._mapped_tables.keys(): logger.debug('Mapping table: {}'.format(key)) self._mapped_tables[key] = table_mapper() self._mapped_tables[key].map(self) @property def engine(self): """ SQLAlchemy Engine object property """ self._map_db_tables() return self._engine def session(self): """ Database session context manager """ return SessionContextManager(self) def init_database(self): # # Create tables # self._map_db_tables() try: self._metadata.create_all(self.engine) except Exception: self.getLogger().exception('SQLAlchemy raised exception') raise DbError('Check database settings or credentials') @property def metadata(self): return self._metadata def __getDbEngineURI(self): dbPort = self._dbConfig['port'] dbHost = self._dbConfig['host'] engine = self._dbConfig['engine'] dbUser = self._dbConfig['username'] dbPassword = self._dbConfig['password'] if engine == 'sqlite': engineURI = 'sqlite:///%s' % (self._dbConfig['path']) else: if dbUser is not None: if dbPassword is not None: userspec = '%s:%s' % (dbUser, dbPassword) else: userspec = dbUser else: userspec = None if dbPort is not None: hostspec = '%s:%s' % (dbHost, dbPort) else: hostspec = dbHost if userspec is not None: engineURI = '%s://%s@%s/%s' % (engine, userspec, hostspec, self._cm.getDbSchema()) else: engineURI = '%s://%s/%s' % (engine, hostspec, self._cm.getDbSchema()) return engineURI def _getDefaultDbEngine(self): \ # pylint: disable=no-self-use return 'sqlite' def _getDefaultDbHost(self): \ # pylint: disable=no-self-use return 'localhost' def _getDefaultDbPort(self, engine): \ # pylint: disable=no-self-use # MySQL default port if engine == 'mysql': return 3306 return None def _getDefaultDbUserName(self): return self._cm.getDbUser() def _getDefaultDbPassword(self): if os.path.exists(self._cm.getDbPasswordFile()): with open(self._cm.getDbPasswordFile()) as fp: dbPassword = fp.read() else: dbPassword = None return dbPassword def _refreshDbConfig(self, cfg=None): dbConfig = {} if cfg is None: cfg = configparser.ConfigParser() cfg.read(os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini')) # Database engine val = cfg.get('database', 'engine').strip().lower() \ if cfg.has_option('database', 'engine') else \ self._getDefaultDbEngine() dbConfig['engine'] = val if dbConfig['engine'] == 'sqlite': # If database is sqlite, read the path dbConfig['path'] = cfg.get('database', 'path') \ if cfg.has_section('database') and \ cfg.has_option('database', 'path') else \ os.path.join(self._cm.getEtcDir(), self._cm.getDbSchema() + '.sqlite') # Database host val = cfg.get('database', 'host') \ if cfg.has_option('database', 'host') else \ self._getDefaultDbHost() dbConfig['host'] = val # Database port val = cfg.get('database', 'port') \ if cfg.has_option('database', 'port') else None dbConfig['port'] = val if val else self._getDefaultDbPort( engine=dbConfig['engine']) # Database username val = cfg.get('database', 'username') \ if cfg.has_option('database', 'username') \ else self._getDefaultDbUserName() dbConfig['username'] = val # Database password val = cfg.get('database', 'password') \ if cfg.has_option('database', 'password') \ else self._getDefaultDbPassword() dbConfig['password'] = val return dbConfig def get_backend_opts(self): \ # pylint: disable=no-self-use return { 'mysql_engine': 'InnoDB', } def getMetadataTable(self, table): return self._metadata.tables[table] def openSession(self): """ Open db session. """ session_factory = sqlalchemy.orm.sessionmaker(bind=self.engine) self.Session = sqlalchemy.orm.scoped_session(session_factory) return self.Session() def closeSession(self): """Close scoped_session.""" self.Session.remove()
class TortugaDeployer: \ # pylint: disable=too-many-public-methods def __init__(self, logger, cmdline_options=None): self._cm = ConfigManager() self._logger = logger self._osObjectFactory = osUtility.getOsObjectFactory() self._settings = self.__load_settings(cmdline_options) self._settings['installer_software_profile'] = 'Installer' self._settings['installer_hardware_profile'] = 'Installer' self._settings['eulaAccepted'] = False self._settings['fqdn'] = getfqdn() self._settings['osInfo'] = getOsInfo() self._forceCleaning = False self._depotCreated = False fsManager = self._osObjectFactory.getOsFileSystemManager() self._lockFilePath = os.path.join( fsManager.getOsLockFilePath(), 'tortuga-setup') langdomain = 'tortuga-config' localedir = os.path.join(self._cm.getRoot(), 'share', 'locale') if not os.path.exists(localedir): # Try the system path localedir = '/usr/share/locale' gettext.bindtextdomain(langdomain, localedir) gettext.textdomain(langdomain) self.gettext = gettext.gettext self._ = self.gettext self._logger.info('Detected OS: [%s]', self._settings['osInfo']) def __load_settings(self, cmdline_options): settings = dict(list(cmdline_options.items())) default_cfgfile = os.path.join( self._cm.getKitConfigBase(), 'tortuga.ini') if 'inifile' in cmdline_options and \ cmdline_options['inifile'] != default_cfgfile: # Copy configuration specified on command-line to # $TORTUGA_ROOT/config/tortuga.ini self._logger.info( 'Using configuration file [%s]' % (settings['inifile'])) self._logger.info( 'Copying configuration to [%s]' % (default_cfgfile)) if os.path.exists(default_cfgfile): # Back up existing 'tortuga.ini' shutil.move(default_cfgfile, default_cfgfile + '.orig') shutil.copyfile(cmdline_options['inifile'], default_cfgfile) settings['inifile'] = default_cfgfile cfg = configparser.ConfigParser() cfg.read(settings['inifile']) settings['timezone'] = '' settings['utc'] = False settings['keyboard'] = 'us' settings['language'] = 'en_US.UTF-8' # Get database setting value = cfg.get('database', 'engine') \ if cfg.has_section('database') and \ cfg.has_option('database', 'engine') else None if value and value not in ('mysql', 'sqlite'): raise InvalidArgument( 'Unsupported database engine [%s]' % (value)) settings['database'] = { 'engine': value if value else 'mysql' } # Get depot directory if cfg.has_section('installer') and \ cfg.has_option('installer', 'depotpath'): settings['depotpath'] = cfg.get('installer', 'depotpath') # For consistency's sake... self._cm.setDepotDir(settings['depotpath']) else: settings['depotpath'] = self._cm.getDepotDir() # Internal web port settings['intWebPort'] = cfg.getint('installer', 'intWebPort') \ if cfg.has_section('installer') and \ cfg.has_option('installer', 'intWebPort') else \ self._cm.getIntWebPort() self._cm.setIntWebPort(settings['intWebPort']) # Admin port settings['adminPort'] = cfg.getint('installer', 'adminPort') \ if cfg.has_section('installer') and \ cfg.has_option('installer', 'adminPort') else \ self._cm.getAdminPort() self._cm.setAdminPort(settings['adminPort']) # IntWebServicePort settings['intWebServicePort'] = cfg.getint( 'installer', 'intWebServicePort') \ if cfg.has_section('installer') and \ cfg.has_option('installer', 'intWebServicePort') else \ self._cm.getIntWebServicePort() self._cm.setIntWebServicePort(settings['intWebServicePort']) return settings def _get_setting(self, name, section=None): if section and section in self._settings: return self._settings[section][name] \ if name in self._settings[section] else None return self._settings[name] if name in self._settings else None def eout(self, message, *args): """ Output messages to STDERR with Internationalization. Additional arguments will be used to substitute variables in the message output """ if args: mesg = self.gettext(message) % args else: mesg = self.gettext(message) sys.stderr.write(mesg) def out(self, message, *args): """ Output messages to STDOUT with Internationalization. Additional arguments will be used to substitute variables in the message output """ if args: mesg = self.gettext(message) % args else: mesg = self.gettext(message) sys.stdout.write(mesg) def prompt(self, default_value, auto_answer_default_value, text_list, question, tag=None, section=None, isPassword=False): """Generic user prompting routine""" resp_value = None bDefaults = self._settings['defaults'] if tag: resp_value = self._get_setting(tag, section=section) if not resp_value and bDefaults: # Use the default value default_value = auto_answer_default_value elif bDefaults: default_value = auto_answer_default_value if text_list: self.out('\n') for line in text_list: self.out(line + '\n') if default_value and not isPassword: self.out('\n%s [%s]: ' % (question, default_value)) else: self.out('\n%s: ' % (question)) if bDefaults or resp_value: if resp_value: value = resp_value else: value = auto_answer_default_value if not isPassword: self.out('%s\n' % value) else: if isPassword: import getpass value = getpass.getpass('').strip() else: value = input('').strip() if not value: value = default_value return value def checkPreInstallConfiguration(self): # pylint: disable=no-self-use """ Raises: InvalidMachineConfiguration """ # Check for existence of /etc/hosts if not os.path.exists('/etc/hosts'): raise InvalidMachineConfiguration( '/etc/hosts file is missing. Unable to proceed with' ' installation') def preInstallPrep(self): bAcceptEula = self._settings['acceptEula'] license_file = ' %s/LICENSE' % (self._cm.getEtcDir()) print() if bAcceptEula: cmd = 'cat %s\n' % (license_file) os.system(cmd) else: cmd = 'more %s\n' % (license_file) print("To install Tortuga you must read and agree to " "the following EULA.") print("Press 'Enter' to continue...") input('') os.system(cmd) print() while True: print('Do you agree? [Yes / No]', end=' ') answer = input('').lower() if answer not in ['yes', 'no', 'y', 'n']: print('Invalid response. Please respond \'Yes\'' ' or \'No\'') continue break if answer[0] == 'n': raise EulaAcceptanceRequired( 'You must accept the EULA to install Tortuga') self._settings['eulaAccepted'] = \ 'Accepted on: %s local machine time' % (time.ctime()) # Restore resolv.conf if we have a backup if osUtility.haveBackupFile('/etc/resolv.conf'): osUtility.restoreFile('/etc/resolv.conf') def _runCommandWithSpinner(self, cmd, statusMsg, logFileName): self._logger.debug( '_runCommandWithSpinner(cmd=[%s], logFileName=[%s])' % ( cmd, logFileName)) self.out(statusMsg + ' ') # Open the log file in unbuffered mode fpOut = open(logFileName, 'ab', 0) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, close_fds=True) for i in itertools.cycle(['/', '-', '\\', '|']): buf = p.stdout.readline() sys.stdout.write('') sys.stdout.flush() if not buf: break fpOut.write(buf) sys.stdout.write(i) sys.stdout.flush() sys.stdout.write(' ') self.out('done.\n') retval = p.wait() fpOut.close() return retval def puppetApply(self): ''' Complete the installer configuration by running against the previously installed Puppet master. Display a spinner while Puppet runs. ''' self._logger.info('Running Puppet for post-configuration') logFileName = '/tmp/tortuga_setup.log' cmd = ('/opt/puppetlabs/bin/puppet agent --color false --onetime' ' --no-daemonize --detailed-exitcodes --verbose 2>&1') retval = self._runCommandWithSpinner( cmd, statusMsg=( '\nCompleting installer configuration.' ' Please wait...'), logFileName=logFileName) if retval not in (0, 2): # Puppet can return a non-zero return code, even if it was # successful. errmsg = 'Puppet post-configuration failed (see log file %s)' % ( logFileName) self._logger.error(errmsg) self.out(errmsg + '\n') raise Exception(errmsg) self._logger.info('Puppet post-configuration completed') def startSetup(self): # If force was specified clean first and then run... bForce = self._settings['force'] if bForce: self._forceCleaning = True self.out( '--force option specified. Cleaning previous' ' installation.\n') self.cleanup() self._forceCleaning = False if os.path.exists(self._lockFilePath): raise SoftwareAlreadyDeployed( "\ntortuga-setup has already been run.\n\n" "Use --force option to force reinstallation.") open(self._lockFilePath, 'w').close() self.out('Tortuga Setup\n') def getClusterConfig(self): sysManager = self._osObjectFactory.getOsSysManager() self._settings['timezone'], self._settings['utc'] = \ sysManager.findTimeInfo() self._settings['keyboard'] = sysManager.findKeyboard() self._settings['language'] = sysManager.findLanguage() self.out(_('\nStarting Tortuga setup...\n')) # Ports configuration if not self._settings['defaults']: intWebPort, adminPort, intWebServicePort = self.configurePorts() self._cm.setIntWebPort(intWebPort) self._cm.setAdminPort(adminPort) self._cm.setIntWebServicePort(intWebServicePort) self._settings['intWebPort'] = intWebPort self._settings['adminPort'] = adminPort self._settings['intWebServicePort'] = intWebServicePort # Admin username and password self._settings['adminUsername'], \ self._settings['adminPassword'] = self.promptForAdminCredentials() def prepDepot(self): depotpath = None if not self._settings['defaults']: self.out( _('Tortuga requires a directory for storage of OS' ' distribution media and other files required for' ' node provisioning.\n\n')) while not depotpath: if self._settings['defaults']: response = self._settings['depotpath'] else: try: response = input( 'Please enter a depot path (Ctrl-C to interrupt)' ' [%s]: ' % (self._settings['depotpath'])) except KeyboardInterrupt: raise InvalidArgument(_('Aborted by user.')) if not response: response = self._settings['depotpath'] if not response.startswith('/'): errmsg = 'Depot path must be fully-qualified' if not self._settings['defaults']: self.out('Error: %s\n' % (errmsg)) continue raise InvalidArgument(errmsg) if response == '/': errmsg = 'Depot path cannot be system root directory' if not self._settings['defaults']: self.out(_('Error: %s\n' % (errmsg))) continue raise InvalidArgument(errmsg) if os.path.exists(response): if not self._settings['force']: if not self._settings['defaults']: self.out( _('Directory [%s] already exists. Do you wish to' ' remove it [N/y]? ') % (response)) remove_response = input('') if not remove_response or \ remove_response[0].lower() == 'n': continue_response = input( 'Do you wish to continue [N/y]? ') if continue_response and \ continue_response[0].lower() == 'y': continue raise InvalidArgument(_('Aborted by user.')) else: raise InvalidArgument( _('Existing depot directory [%s] will not be' ' removed.') % (response)) else: self.out( _('\nRemoving existing depot directory [%s]... ') % ( response)) depotpath = response tortugaSubprocess.executeCommand( 'rm -rf %s/*' % (depotpath)) self.out(_('done.\n')) else: depotpath = response self._settings['depotpath'] = depotpath self._cm.setDepotDir(self._settings['depotpath']) def _portPrompt(self, promptStr, defaultValue): while True: tmpPort = self.prompt( defaultValue, defaultValue, None, promptStr) try: tmpPort = int(tmpPort) if tmpPort <= 0 or tmpPort > 65535: raise ValueError('Port must be between 1 and 65535') # Success break except ValueError as ex: self.out('Error: ' + str(ex) + '\n') return tmpPort def configurePorts(self): reconfigurePorts = self.prompt( 'N', 'N', [ 'The following ports will be used by Tortuga:' '', ' +-----------------------------+-------+', ' | Description | Port |', ' +-----------------------------+-------+', ' | Internal webserver | %5d |' % ( self._settings['intWebPort']), ' | SSL webservice daemon | %5d |' % ( self._settings['adminPort']), ' | Local webservice daemon | %5d |' % ( self._settings['intWebServicePort']), ' +-----------------------------+-------+' ], 'Do you wish to change the default configuration [N/y]?') if not reconfigurePorts or reconfigurePorts[0].lower() == 'n': return self._settings['intWebPort'], \ self._settings['adminPort'], \ self._settings['intWebServicePort'] # Internal web server port intWebPort = self._portPrompt( 'Enter port for internal webserver', self._settings['intWebPort']) # SSL webservice daemon port adminPort = self._portPrompt( 'Enter port for SSL webservice daemon', self._settings['adminPort']) # Local webservice daemon port intWebServicePort = self._portPrompt( 'Enter port for local webservice daemon', self._settings['intWebServicePort']) return intWebPort, adminPort, intWebServicePort def _removePackageSources(self): pkgManager = self._osObjectFactory.getOsPackageManager() for pkgSrcName in pkgManager.getPackageSourceNames(): self._logger.info( 'Removing package source [%s]' % (pkgSrcName)) pkgManager.removePackageSource(pkgSrcName) def _disableTortugaws(self): self.out(' * Disabling Tortuga webservice\n') _tortugaWsManager = self._osObjectFactory.getTortugawsManager() serviceName = _tortugaWsManager.getServiceName() _osServiceManager = getOsObjectFactory().getOsServiceManager() try: _osServiceManager.stop(serviceName) except CommandFailed: pass def cleanup(self): # If possible, remove any package sources we added self._removePackageSources() osUtility.removeFile(self._lockFilePath) osUtility.removeFile(self._cm.getProfileNiiFile()) # Turn off the webservice daemon self._disableTortugaws() # Restore resolv.conf if osUtility.haveBackupFile('/etc/resolv.conf'): osUtility.restoreFile('/etc/resolv.conf') # Drop database dbManager = self._osObjectFactory.getOsApplicationManager( self._settings['database']['engine']) try: dbSchema = self._cm.getDbSchema() self.out(' * Removing database [%s]\n' % (dbSchema)) dbManager.destroyDb(dbSchema) except Exception as ex: # pylint: disable=broad-except self._logger.exception( 'Could not destroy existing db: {}'.format(ex)) # Remove DB password file osUtility.removeFile(self._cm.getDbPasswordFile()) # Remove CFM secret cfmSecretFile = self._cm.getCfmSecretFile() if os.path.exists(cfmSecretFile): osUtility.removeFile(self._cm.getCfmSecretFile()) # Generic cleanup osUtility.removeLink('/etc/tortuga-release') # Cleanup or remove depot directory errmsg = 'Removing contents of [%s]' % (self._settings['depotpath']) self._logger.debug(errmsg) if self._depotCreated: self.out(' * %s\n' % (errmsg)) osUtility.removeDir(self._settings['depotpath']) else: if self._settings['depotpath']: self.out(' * %s\n' % (errmsg)) tortugaSubprocess.executeCommand( 'rm -rf %s/*' % (self._settings['depotpath'])) self.out('\n') if not self._forceCleaning: self.out('Consult log(s) for further details.\n') self._logger.error('Installation failed') def runSetup(self): """ Installer setup. """ self.checkPreInstallConfiguration() # Do not run cleanup if this fails. self.startSetup() try: self.preInstallPrep() self.getClusterConfig() self.prepDepot() self.preConfig() self.pre_init_db() self.puppetBootstrap() dbm, session = self.initDatabase() try: self.createAdminUser( session, self._settings['adminUsername'], self._settings['adminPassword']) self.installKits(dbm) self.enableComponents(session) finally: dbm.closeSession() self.puppetApply() self.out('\nTortuga installation completed successfully!\n\n') print('Run \"exec -l $SHELL\" to initialize Tortuga environment\n') except Exception: # pylint: disable=broad-except self._logger.exception('Fatal error occurred during setup') raise TortugaException('Installation failed') def _generate_db_password(self): """ Generate a database password. """ # # Because Apache httpd server is not installed at the time this # runs, we cannot set the ownership of this file to be 'apache' # (which is necessary for the Tortuga webservice). # # Set ownership of file to root:puppet. # # When the Puppet bootstrap runs, it changes the ownership to # 'apache:puppet' and everybody is happy! # puppet_user = pwd.getpwnam('puppet') gid = puppet_user[3] self._generate_password_file(self._cm.getDbPasswordFile(), gid=gid) def _generate_redis_password(self): """ Generate a password for Redis. """ # # Puppet needs read access to this file so that it can use it for # writing the redis config file. # puppet_user = pwd.getpwnam('puppet') gid = puppet_user[3] self._generate_password_file(self._cm.getRedisPasswordFile(), gid=gid) def _generate_password_file(self, file_name: str, password_length: int = 32, uid: int = 0, gid: int = 0, mode: int = 0o440): """ Generate a password in a file. :param file_name: the name of the file in which the password will be stored :param password_length: the length of the password, default = 32 :param uid: the uid (owner) of the file, default = 0 :param gid: the gid (group) of the file, default = 0 :param mode: the file perms, default 0440 """ password = self._generate_password(password_length) with open(file_name, 'w') as fp: fp.write(password) os.chown(file_name, uid, gid) os.chmod(file_name, mode) def _generate_password(self, length: int = 8) -> str: """ Generate a password. :param length: the length of the password :return: the generated password """ chars = string.ascii_letters + string.digits return ''.join([random.choice(chars) for _ in range(length)]) def preConfig(self): # Create default hieradata directory hieraDataDir = '/etc/puppetlabs/code/environments/production/data' if not os.path.exists(hieraDataDir): os.makedirs(hieraDataDir) # Derive host name of puppet master from FQDN fqdn = self._settings['fqdn'] configDict = { 'version': 5, 'DNSZone': 'private', 'puppet_server': fqdn, 'depot': self._settings['depotpath'], } with open(os.path.join(hieraDataDir, 'tortuga-common.yaml'), 'wb') as fp: fp.write( yaml.safe_dump( configDict, explicit_start=True, default_flow_style=False).encode()) self._generate_db_password() self._generate_redis_password() def pre_init_db(self): # If using 'mysql' as the database backend, we need to install the # puppetlabs-mysql Puppet module prior to bootstrapping. This used # to be done in 'install-tortuga.sh' if self._settings['database']['engine'] == 'mysql': print('\nUsing MySQL as backing database.') puppet_module = 'puppetlabs-mysql' logmsg = f'Installing \'{puppet_module}\' module' self._logger.debug(logmsg) print(f'\n{logmsg}...', end='') cmd = ('/opt/puppetlabs/bin/puppet module install' f' --color false {puppet_module}') tortugaSubprocess.executeCommand(cmd) print('done.') def puppetBootstrap(self): localPuppetRoot = os.path.join(self._cm.getEtcDir(), 'puppet') logFileName = '/tmp/bootstrap.log' puppet_server = self._settings['fqdn'] # Bootstrap using Puppet cmd = ('/opt/puppetlabs/bin/puppet apply --verbose' ' --detailed-exitcodes' ' --execute "class { \'tortuga::installer\':' ' puppet_server => \'%s\',' '}"' % (puppet_server) ) retval = self._runCommandWithSpinner( cmd, '\nPerforming pre-configuration... Please wait...', logFileName=logFileName) if retval not in (0, 2): # Puppet can return a non-zero return code, even if it was # successful. self._logger.debug( 'Puppet pre-configuration returned non-zero' ' return code [%d]' % (retval)) errmsg = 'Puppet bootstrap failed (see log file %s)' % ( logFileName) self._logger.error(errmsg) raise Exception(errmsg) self._logger.debug('Puppet pre-configuration completed') def initDatabase(self) -> Tuple[Any, Session]: msg = _('Initializing database') self._logger.info(msg) print_('\n' + msg + '... ', end='') # This cannot be a global import since the database configuration # may be set in this script. from tortuga.db.dbManager import DbManager dbm = DbManager() # create database dbm.init_database() session = dbm.openSession() # Prime the database previously created as part of the bootstrap try: dbUtility.primeDb(session, self._settings) dbUtility.init_global_parameters(session, self._settings) print_(_('done')) session.commit() except Exception as exc: # pylint: disable=broad-except session.rollback() print_(_('failed.')) print_(_('Exception raised initializing database:') + ' {0}'.format(exc), file=sys.stderr) self._logger.debug('Done initializing database') return dbm, session def installKits(self, dbm): self._logger.info('Installing kits') self.out('\n' + _('Installing kits') + '...\n') kitApi = KitApi() # Iterate over the glob of 'kits-*.tar.bz2' kitFileGlob = '%s/kits/kit-*.tar.bz2' % (self._cm.getRoot()) # Split comma-separated list of kits to skip installing. Sorry, you # cannot skip installing the base kit. val = self._settings['skip_kits'] \ if 'skip_kits' in self._settings else '' skip_kits = set([ item for item in val.split(',') if item != 'base']) \ if val else set() for kitPackage in glob.glob(kitFileGlob): try: kit = get_metadata_from_archive(kitPackage) except KitNotFound: msg = 'Kit [%s] is malformed/invalid. Skipping.' % ( os.path.basename(kitPackage)) self._logger.error(msg) self.out(' %s\n' % (msg)) continue if kit['name'] in skip_kits: msg = 'Kit [%s] installation skipped.' % (kit['name']) self.out(' %s\n' % (msg)) self._logger.info(msg) continue try: kitApi.installKitPackage(dbm, kitPackage) except EulaAcceptanceRequired: msg = 'Kit [%s] requires EULA acceptance. Skipping.' % ( kitPackage) self.out(' %s\n' % (msg)) self._logger.info(msg) continue self.out(' - %s installed.\n' % (kit['name'])) self._logger.info('Kit [%s] installed' % (kit['name'])) self._logger.info('Done installing kits') load_kits() def enableComponents(self, session: Session): """ Raises: ConfigurationError """ self._logger.info('Enabling \'installer\' component') base_kit = KitApi().getKit(session, 'base') enabledComponents = ['installer'] # get list of components from 'base' kit components = [c for c in base_kit.getComponentList() if c.getName() in enabledComponents] installerNode = NodeApi().getInstallerNode(session) for component in components: SoftwareProfileApi().enableComponent( session, installerNode.getSoftwareProfile().getName(), base_kit.getName(), base_kit.getVersion(), base_kit.getIteration(), component.getName(), compVersion=component.getVersion(), ) def promptForAdminCredentials(self): # Get admin username and password for use with web service if self._settings['defaults']: self.out(_('\nUsing default Tortuga admin user name/password.\n')) return 'admin', 'password' username = password = None # Administrator username while True: username = self.prompt( 'admin', 'admin', ['Enter name for Tortuga admin user.', 'This user is not associated with any system user.'], 'Admin user name') if len(username) > 3: break self.out('Admin user name must be at least 4 characters.') # Administrator password while True: password = self.prompt( '', 'password', ['Enter password for Tortuga admin user.'], 'Admin password', None, None, True) if len(password) < 4: self.out('Admin password must be at least 4 characters.') continue confirmPassword = self.prompt( '', 'password', ['Confirm admin password.'], 'Confirm password', None, None, True) if confirmPassword == password: self.out('\n') break self.out('Passwords did not match.') return username, password def createAdminUser(self, session: Session, username, password): msg = _('Adding administrative user') self._logger.info(msg) self.out('\n' + msg + '... ') AdminApi().addAdmin( session, username, password, False, description='Added by tortuga-setup') self.out(_('done.') + '\n')
class DbManager(TortugaObjectManager): """ Class for db management. :param engine: a SQLAlchemy database engine instance :param init: a flag that is set when the database has not yet been initialized. If this flag is set, not attempts will be made to load/map kit tables. This flag is cleared once the database has been initialized. """ def __init__(self, engine=None): super().__init__() if not engine: self._cm = ConfigManager() self._dbConfig = self._refreshDbConfig() engineURI = self.__getDbEngineURI() if self._dbConfig['engine'] == 'sqlite' and \ not os.path.exists(self._dbConfig['path']): # Ensure SQLite database file is created with proper permissions fd = os.open( self._dbConfig['path'], os.O_CREAT, mode=0o600) os.close(fd) self._engine = sqlalchemy.create_engine(engineURI) else: self._engine = engine self.Session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=self.engine)) def _register_database_tables(self): for kit_installer_class in get_all_kit_installers(): kit_installer = kit_installer_class() kit_installer.register_database_tables() @property def engine(self): """ SQLAlchemy Engine object property """ self._register_database_tables() return self._engine def session(self): """ Database session context manager """ return SessionContextManager(self) def init_database(self): # # Create tables # self._register_database_tables() try: ModelBase.metadata.create_all(self.engine) except Exception: self._logger.exception('SQLAlchemy raised exception') raise DbError('Check database settings or credentials') @property def metadata(self): return self._metadata def __getDbEngineURI(self): dbPort = self._dbConfig['port'] dbHost = self._dbConfig['host'] engine = self._dbConfig['engine'] dbUser = self._dbConfig['username'] dbPassword = self._dbConfig['password'] if engine == 'sqlite': engineURI = 'sqlite:///%s' % (self._dbConfig['path']) else: if dbUser is not None: if dbPassword is not None: userspec = '%s:%s' % (dbUser, dbPassword) else: userspec = dbUser else: userspec = None if dbPort is not None: hostspec = '%s:%s' % (dbHost, dbPort) else: hostspec = dbHost engineURI = f'{engine}+pymysql' if engine == 'mysql' else engine engineURI += '://' if userspec is not None: engineURI += f'{userspec}@' engineURI += f'{hostspec}' + '/{}'.format(self._cm.getDbSchema()) return engineURI def _getDefaultDbEngine(self): \ # pylint: disable=no-self-use return 'sqlite' def _getDefaultDbHost(self): \ # pylint: disable=no-self-use return 'localhost' def _getDefaultDbPort(self, engine): \ # pylint: disable=no-self-use # MySQL default port if engine == 'mysql': return 3306 return None def _getDefaultDbUserName(self): return self._cm.getDbUser() def _getDefaultDbPassword(self): if os.path.exists(self._cm.getDbPasswordFile()): with open(self._cm.getDbPasswordFile()) as fp: dbPassword = fp.read() else: dbPassword = None return dbPassword def _refreshDbConfig(self, cfg=None): dbConfig = {} if cfg is None: cfg = configparser.ConfigParser() cfg.read(os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini')) # Database engine val = cfg.get('database', 'engine').strip().lower() \ if cfg.has_option('database', 'engine') else \ self._getDefaultDbEngine() dbConfig['engine'] = val if dbConfig['engine'] == 'sqlite': # If database is sqlite, read the path dbConfig['path'] = cfg.get('database', 'path') \ if cfg.has_section('database') and \ cfg.has_option('database', 'path') else \ os.path.join(self._cm.getEtcDir(), self._cm.getDbSchema() + '.sqlite') # Database host val = cfg.get('database', 'host') \ if cfg.has_option('database', 'host') else \ self._getDefaultDbHost() dbConfig['host'] = val # Database port val = cfg.get('database', 'port') \ if cfg.has_option('database', 'port') else None dbConfig['port'] = val if val else self._getDefaultDbPort( engine=dbConfig['engine']) # Database username val = cfg.get('database', 'username') \ if cfg.has_option('database', 'username') \ else self._getDefaultDbUserName() dbConfig['username'] = val # Database password val = cfg.get('database', 'password') \ if cfg.has_option('database', 'password') \ else self._getDefaultDbPassword() dbConfig['password'] = val return dbConfig def get_backend_opts(self): \ # pylint: disable=no-self-use return { 'mysql_engine': 'InnoDB', } def getMetadataTable(self, table): return self._metadata.tables[table] def openSession(self): """ Open db session. """ return self.Session() def closeSession(self): """Close scoped_session.""" self.Session.remove()
class DbManager(TortugaObjectManager): """ Class for db management. :param engine: a SQLAlchemy database engine instance :param init: a flag that is set when the database has not yet been initialized. If this flag is set, not attempts will be made to load/map kit tables. This flag is cleared once the database has been initialized. """ def __init__(self, engine=None): super().__init__() if not engine: self._cm = ConfigManager() engine = self._cm.getDbEngine() schema = self._cm.getDbSchema() engineURI = self.__getDbEngineURI() if engine == 'sqlite' and not os.path.exists(schema): # Ensure SQLite database file is created with proper # permissions fd = os.open(schema, os.O_CREAT, mode=0o600) os.close(fd) self._engine = sqlalchemy.create_engine(engineURI, pool_size=100, max_overflow=100, pool_pre_ping=True) else: self._engine = engine self.Session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=self.engine)) def _register_database_tables(self): for kit_installer_class in get_all_kit_installers(): kit_installer = kit_installer_class() kit_installer.register_database_tables() @property def engine(self): """ SQLAlchemy Engine object property """ self._register_database_tables() return self._engine def session(self): """ Database session context manager """ return SessionContextManager(self) def init_database(self): # # Create tables # self._register_database_tables() ModelBase.metadata.create_all(self.engine) @property def metadata(self): return self._metadata def __getDbEngineURI(self): engine = self._cm.getDbEngine() schema = self._cm.getDbSchema() driver = '' host = '' port = '' user = '' password = '' if engine == 'mysql': driver = "+pymysql" host = self._cm.getDbHost() port = self._cm.getDbPort() user = self._cm.getDbUser() password = self._cm.getDbPassword() userspec = '' if user: if password: userspec = '{}:{}@'.format(user, password) else: userspec = '{}@'.format(user) hostspec = '' if host: if port: hostspec = '{}:{}'.format(host, port) else: hostspec = host engineURI = "{}{}://{}{}/{}".format(engine, driver, userspec, hostspec, schema) return engineURI def getMetadataTable(self, table): return self._metadata.tables[table] def openSession(self): """ Open db session. """ return self.Session() def closeSession(self): """Close scoped_session.""" self.Session.remove()