def get_polling_interval(logger=None):
"""
Return the periodic polling interval referencing a configuraiton
file and the then defaulting to a reasonable value if not present.
"""
interval = SPOT_INSTANCE_POLLING_INTERVAL
config_manager = ConfigManager()
base_dir = config_manager.getKitConfigBase()
try:
config_parser = configparser.ConfigParser()
config_parser.read(os.path.join(base_dir, CONFIG_FILE))
interval = int(config_parser.get(CONFIG_SECTION, CONFIG_INTERVAL_SECS))
except Exception as ex: # pylint: disable=broad-except
if logger:
logger.error(
'Unable to load configuration file [%s]. Using defaults.', ex)
return interval
class DbManagerBase(TortugaObjectManager):
"""
Class for db management.
:param engine: a SQLAlchemy database engine instance
:param init: a flag that is set when the database has not yet
been initialized. If this flag is set, not attempts
will be made to load/map kit tables. This flag is
cleared once the database has been initialized.
"""
def __init__(self, engine=None):
super(DbManagerBase, self).__init__()
self.Session = None
if not engine:
self._cm = ConfigManager()
self._dbConfig = self._refreshDbConfig()
engineURI = self.__getDbEngineURI()
if self._dbConfig['engine'] == 'mysql':
# Set default SQLAlchemy engine arguments for MySQL
kwargs = {
'pool_size': 10,
'max_overflow': 2,
'pool_recycle': 600,
'echo': False,
'pool_timeout': 60,
}
else:
kwargs = {}
self._engine = sqlalchemy.create_engine(engineURI, **kwargs)
else:
self._engine = engine
self._metadata = sqlalchemy.MetaData(self._engine)
self._mapped_tables = {}
def _map_db_tables(self):
#
# Make sure all kit table mappers have been registered
#
load_kits()
for kit_installer_class in get_all_kit_installers():
kit_installer = kit_installer_class()
kit_installer.register_database_table_mappers()
#
# Map all tables that haven't yet been mapped
#
for table_mapper in get_all_table_mappers():
key = table_mapper.__name__
if key not in self._mapped_tables.keys():
logger.debug('Mapping table: {}'.format(key))
self._mapped_tables[key] = table_mapper()
self._mapped_tables[key].map(self)
@property
def engine(self):
"""
SQLAlchemy Engine object property
"""
self._map_db_tables()
return self._engine
def session(self):
"""
Database session context manager
"""
return SessionContextManager(self)
def init_database(self):
#
# Create tables
#
self._map_db_tables()
try:
self._metadata.create_all(self.engine)
except Exception:
self.getLogger().exception('SQLAlchemy raised exception')
raise DbError('Check database settings or credentials')
@property
def metadata(self):
return self._metadata
def __getDbEngineURI(self):
dbPort = self._dbConfig['port']
dbHost = self._dbConfig['host']
engine = self._dbConfig['engine']
dbUser = self._dbConfig['username']
dbPassword = self._dbConfig['password']
if engine == 'sqlite':
engineURI = 'sqlite:///%s' % (self._dbConfig['path'])
else:
if dbUser is not None:
if dbPassword is not None:
userspec = '%s:%s' % (dbUser, dbPassword)
else:
userspec = dbUser
else:
userspec = None
if dbPort is not None:
hostspec = '%s:%s' % (dbHost, dbPort)
else:
hostspec = dbHost
if userspec is not None:
engineURI = '%s://%s@%s/%s' % (engine, userspec, hostspec,
self._cm.getDbSchema())
else:
engineURI = '%s://%s/%s' % (engine, hostspec,
self._cm.getDbSchema())
return engineURI
def _getDefaultDbEngine(self): \
# pylint: disable=no-self-use
return 'sqlite'
def _getDefaultDbHost(self): \
# pylint: disable=no-self-use
return 'localhost'
def _getDefaultDbPort(self, engine): \
# pylint: disable=no-self-use
# MySQL default port
if engine == 'mysql':
return 3306
return None
def _getDefaultDbUserName(self):
return self._cm.getDbUser()
def _getDefaultDbPassword(self):
if os.path.exists(self._cm.getDbPasswordFile()):
with open(self._cm.getDbPasswordFile()) as fp:
dbPassword = fp.read()
else:
dbPassword = None
return dbPassword
def _refreshDbConfig(self, cfg=None):
dbConfig = {}
if cfg is None:
cfg = configparser.ConfigParser()
cfg.read(os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini'))
# Database engine
val = cfg.get('database', 'engine').strip().lower() \
if cfg.has_option('database', 'engine') else \
self._getDefaultDbEngine()
dbConfig['engine'] = val
if dbConfig['engine'] == 'sqlite':
# If database is sqlite, read the path
dbConfig['path'] = cfg.get('database', 'path') \
if cfg.has_section('database') and \
cfg.has_option('database', 'path') else \
os.path.join(self._cm.getEtcDir(),
self._cm.getDbSchema() + '.sqlite')
# Database host
val = cfg.get('database', 'host') \
if cfg.has_option('database', 'host') else \
self._getDefaultDbHost()
dbConfig['host'] = val
# Database port
val = cfg.get('database', 'port') \
if cfg.has_option('database', 'port') else None
dbConfig['port'] = val if val else self._getDefaultDbPort(
engine=dbConfig['engine'])
# Database username
val = cfg.get('database', 'username') \
if cfg.has_option('database', 'username') \
else self._getDefaultDbUserName()
dbConfig['username'] = val
# Database password
val = cfg.get('database', 'password') \
if cfg.has_option('database', 'password') \
else self._getDefaultDbPassword()
dbConfig['password'] = val
return dbConfig
def get_backend_opts(self): \
# pylint: disable=no-self-use
return {
'mysql_engine': 'InnoDB',
}
def getMetadataTable(self, table):
return self._metadata.tables[table]
def openSession(self):
""" Open db session. """
session_factory = sqlalchemy.orm.sessionmaker(bind=self.engine)
self.Session = sqlalchemy.orm.scoped_session(session_factory)
return self.Session()
def closeSession(self):
"""Close scoped_session."""
self.Session.remove()
class TortugaDeployer: \
# pylint: disable=too-many-public-methods
def __init__(self, logger, cmdline_options=None):
self._cm = ConfigManager()
self._logger = logger
self._osObjectFactory = osUtility.getOsObjectFactory()
self._settings = self.__load_settings(cmdline_options)
self._settings['installer_software_profile'] = 'Installer'
self._settings['installer_hardware_profile'] = 'Installer'
self._settings['eulaAccepted'] = False
self._settings['fqdn'] = getfqdn()
self._settings['osInfo'] = getOsInfo()
self._forceCleaning = False
self._depotCreated = False
fsManager = self._osObjectFactory.getOsFileSystemManager()
self._lockFilePath = os.path.join(
fsManager.getOsLockFilePath(), 'tortuga-setup')
langdomain = 'tortuga-config'
localedir = os.path.join(self._cm.getRoot(), 'share', 'locale')
if not os.path.exists(localedir):
# Try the system path
localedir = '/usr/share/locale'
gettext.bindtextdomain(langdomain, localedir)
gettext.textdomain(langdomain)
self.gettext = gettext.gettext
self._ = self.gettext
self._logger.info('Detected OS: [%s]', self._settings['osInfo'])
def __load_settings(self, cmdline_options):
settings = dict(list(cmdline_options.items()))
default_cfgfile = os.path.join(
self._cm.getKitConfigBase(), 'tortuga.ini')
if 'inifile' in cmdline_options and \
cmdline_options['inifile'] != default_cfgfile:
# Copy configuration specified on command-line to
# $TORTUGA_ROOT/config/tortuga.ini
self._logger.info(
'Using configuration file [%s]' % (settings['inifile']))
self._logger.info(
'Copying configuration to [%s]' % (default_cfgfile))
if os.path.exists(default_cfgfile):
# Back up existing 'tortuga.ini'
shutil.move(default_cfgfile, default_cfgfile + '.orig')
shutil.copyfile(cmdline_options['inifile'], default_cfgfile)
settings['inifile'] = default_cfgfile
cfg = configparser.ConfigParser()
cfg.read(settings['inifile'])
settings['timezone'] = ''
settings['utc'] = False
settings['keyboard'] = 'us'
settings['language'] = 'en_US.UTF-8'
# Get database setting
value = cfg.get('database', 'engine') \
if cfg.has_section('database') and \
cfg.has_option('database', 'engine') else None
if value and value not in ('mysql', 'sqlite'):
raise InvalidArgument(
'Unsupported database engine [%s]' % (value))
settings['database'] = {
'engine': value if value else 'mysql'
}
# Get depot directory
if cfg.has_section('installer') and \
cfg.has_option('installer', 'depotpath'):
settings['depotpath'] = cfg.get('installer', 'depotpath')
# For consistency's sake...
self._cm.setDepotDir(settings['depotpath'])
else:
settings['depotpath'] = self._cm.getDepotDir()
# Internal web port
settings['intWebPort'] = cfg.getint('installer', 'intWebPort') \
if cfg.has_section('installer') and \
cfg.has_option('installer', 'intWebPort') else \
self._cm.getIntWebPort()
self._cm.setIntWebPort(settings['intWebPort'])
# Admin port
settings['adminPort'] = cfg.getint('installer', 'adminPort') \
if cfg.has_section('installer') and \
cfg.has_option('installer', 'adminPort') else \
self._cm.getAdminPort()
self._cm.setAdminPort(settings['adminPort'])
# IntWebServicePort
settings['intWebServicePort'] = cfg.getint(
'installer', 'intWebServicePort') \
if cfg.has_section('installer') and \
cfg.has_option('installer', 'intWebServicePort') else \
self._cm.getIntWebServicePort()
self._cm.setIntWebServicePort(settings['intWebServicePort'])
return settings
def _get_setting(self, name, section=None):
if section and section in self._settings:
return self._settings[section][name] \
if name in self._settings[section] else None
return self._settings[name] if name in self._settings else None
def eout(self, message, *args):
"""
Output messages to STDERR with Internationalization.
Additional arguments will be used to substitute variables in the
message output
"""
if args:
mesg = self.gettext(message) % args
else:
mesg = self.gettext(message)
sys.stderr.write(mesg)
def out(self, message, *args):
"""
Output messages to STDOUT with Internationalization.
Additional arguments will be used to substitute variables in the
message output
"""
if args:
mesg = self.gettext(message) % args
else:
mesg = self.gettext(message)
sys.stdout.write(mesg)
def prompt(self, default_value,
auto_answer_default_value, text_list, question,
tag=None, section=None, isPassword=False):
"""Generic user prompting routine"""
resp_value = None
bDefaults = self._settings['defaults']
if tag:
resp_value = self._get_setting(tag, section=section)
if not resp_value and bDefaults:
# Use the default value
default_value = auto_answer_default_value
elif bDefaults:
default_value = auto_answer_default_value
if text_list:
self.out('\n')
for line in text_list:
self.out(line + '\n')
if default_value and not isPassword:
self.out('\n%s [%s]: ' % (question, default_value))
else:
self.out('\n%s: ' % (question))
if bDefaults or resp_value:
if resp_value:
value = resp_value
else:
value = auto_answer_default_value
if not isPassword:
self.out('%s\n' % value)
else:
if isPassword:
import getpass
value = getpass.getpass('').strip()
else:
value = input('').strip()
if not value:
value = default_value
return value
def checkPreInstallConfiguration(self): # pylint: disable=no-self-use
"""
Raises:
InvalidMachineConfiguration
"""
# Check for existence of /etc/hosts
if not os.path.exists('/etc/hosts'):
raise InvalidMachineConfiguration(
'/etc/hosts file is missing. Unable to proceed with'
' installation')
def preInstallPrep(self):
bAcceptEula = self._settings['acceptEula']
license_file = ' %s/LICENSE' % (self._cm.getEtcDir())
print()
if bAcceptEula:
cmd = 'cat %s\n' % (license_file)
os.system(cmd)
else:
cmd = 'more %s\n' % (license_file)
print("To install Tortuga you must read and agree to "
"the following EULA.")
print("Press 'Enter' to continue...")
input('')
os.system(cmd)
print()
while True:
print('Do you agree? [Yes / No]', end=' ')
answer = input('').lower()
if answer not in ['yes', 'no', 'y', 'n']:
print('Invalid response. Please respond \'Yes\''
' or \'No\'')
continue
break
if answer[0] == 'n':
raise EulaAcceptanceRequired(
'You must accept the EULA to install Tortuga')
self._settings['eulaAccepted'] = \
'Accepted on: %s local machine time' % (time.ctime())
# Restore resolv.conf if we have a backup
if osUtility.haveBackupFile('/etc/resolv.conf'):
osUtility.restoreFile('/etc/resolv.conf')
def _runCommandWithSpinner(self, cmd, statusMsg, logFileName):
self._logger.debug(
'_runCommandWithSpinner(cmd=[%s], logFileName=[%s])' % (
cmd, logFileName))
self.out(statusMsg + ' ')
# Open the log file in unbuffered mode
fpOut = open(logFileName, 'ab', 0)
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, bufsize=1,
close_fds=True)
for i in itertools.cycle(['/', '-', '\\', '|']):
buf = p.stdout.readline()
sys.stdout.write('')
sys.stdout.flush()
if not buf:
break
fpOut.write(buf)
sys.stdout.write(i)
sys.stdout.flush()
sys.stdout.write(' ')
self.out('done.\n')
retval = p.wait()
fpOut.close()
return retval
def puppetApply(self):
'''
Complete the installer configuration by running against the
previously installed Puppet master. Display a spinner while Puppet
runs.
'''
self._logger.info('Running Puppet for post-configuration')
logFileName = '/tmp/tortuga_setup.log'
cmd = ('/opt/puppetlabs/bin/puppet agent --color false --onetime'
' --no-daemonize --detailed-exitcodes --verbose 2>&1')
retval = self._runCommandWithSpinner(
cmd, statusMsg=(
'\nCompleting installer configuration.'
' Please wait...'), logFileName=logFileName)
if retval not in (0, 2):
# Puppet can return a non-zero return code, even if it was
# successful.
errmsg = 'Puppet post-configuration failed (see log file %s)' % (
logFileName)
self._logger.error(errmsg)
self.out(errmsg + '\n')
raise Exception(errmsg)
self._logger.info('Puppet post-configuration completed')
def startSetup(self):
# If force was specified clean first and then run...
bForce = self._settings['force']
if bForce:
self._forceCleaning = True
self.out(
'--force option specified. Cleaning previous'
' installation.\n')
self.cleanup()
self._forceCleaning = False
if os.path.exists(self._lockFilePath):
raise SoftwareAlreadyDeployed(
"\ntortuga-setup has already been run.\n\n"
"Use --force option to force reinstallation.")
open(self._lockFilePath, 'w').close()
self.out('Tortuga Setup\n')
def getClusterConfig(self):
sysManager = self._osObjectFactory.getOsSysManager()
self._settings['timezone'], self._settings['utc'] = \
sysManager.findTimeInfo()
self._settings['keyboard'] = sysManager.findKeyboard()
self._settings['language'] = sysManager.findLanguage()
self.out(_('\nStarting Tortuga setup...\n'))
# Ports configuration
if not self._settings['defaults']:
intWebPort, adminPort, intWebServicePort = self.configurePorts()
self._cm.setIntWebPort(intWebPort)
self._cm.setAdminPort(adminPort)
self._cm.setIntWebServicePort(intWebServicePort)
self._settings['intWebPort'] = intWebPort
self._settings['adminPort'] = adminPort
self._settings['intWebServicePort'] = intWebServicePort
# Admin username and password
self._settings['adminUsername'], \
self._settings['adminPassword'] = self.promptForAdminCredentials()
def prepDepot(self):
depotpath = None
if not self._settings['defaults']:
self.out(
_('Tortuga requires a directory for storage of OS'
' distribution media and other files required for'
' node provisioning.\n\n'))
while not depotpath:
if self._settings['defaults']:
response = self._settings['depotpath']
else:
try:
response = input(
'Please enter a depot path (Ctrl-C to interrupt)'
' [%s]: ' % (self._settings['depotpath']))
except KeyboardInterrupt:
raise InvalidArgument(_('Aborted by user.'))
if not response:
response = self._settings['depotpath']
if not response.startswith('/'):
errmsg = 'Depot path must be fully-qualified'
if not self._settings['defaults']:
self.out('Error: %s\n' % (errmsg))
continue
raise InvalidArgument(errmsg)
if response == '/':
errmsg = 'Depot path cannot be system root directory'
if not self._settings['defaults']:
self.out(_('Error: %s\n' % (errmsg)))
continue
raise InvalidArgument(errmsg)
if os.path.exists(response):
if not self._settings['force']:
if not self._settings['defaults']:
self.out(
_('Directory [%s] already exists. Do you wish to'
' remove it [N/y]? ') % (response))
remove_response = input('')
if not remove_response or \
remove_response[0].lower() == 'n':
continue_response = input(
'Do you wish to continue [N/y]? ')
if continue_response and \
continue_response[0].lower() == 'y':
continue
raise InvalidArgument(_('Aborted by user.'))
else:
raise InvalidArgument(
_('Existing depot directory [%s] will not be'
' removed.') % (response))
else:
self.out(
_('\nRemoving existing depot directory [%s]... ') % (
response))
depotpath = response
tortugaSubprocess.executeCommand(
'rm -rf %s/*' % (depotpath))
self.out(_('done.\n'))
else:
depotpath = response
self._settings['depotpath'] = depotpath
self._cm.setDepotDir(self._settings['depotpath'])
def _portPrompt(self, promptStr, defaultValue):
while True:
tmpPort = self.prompt(
defaultValue, defaultValue, None, promptStr)
try:
tmpPort = int(tmpPort)
if tmpPort <= 0 or tmpPort > 65535:
raise ValueError('Port must be between 1 and 65535')
# Success
break
except ValueError as ex:
self.out('Error: ' + str(ex) + '\n')
return tmpPort
def configurePorts(self):
reconfigurePorts = self.prompt(
'N', 'N', [
'The following ports will be used by Tortuga:'
'',
' +-----------------------------+-------+',
' | Description | Port |',
' +-----------------------------+-------+',
' | Internal webserver | %5d |' % (
self._settings['intWebPort']),
' | SSL webservice daemon | %5d |' % (
self._settings['adminPort']),
' | Local webservice daemon | %5d |' % (
self._settings['intWebServicePort']),
' +-----------------------------+-------+'
], 'Do you wish to change the default configuration [N/y]?')
if not reconfigurePorts or reconfigurePorts[0].lower() == 'n':
return self._settings['intWebPort'], \
self._settings['adminPort'], \
self._settings['intWebServicePort']
# Internal web server port
intWebPort = self._portPrompt(
'Enter port for internal webserver',
self._settings['intWebPort'])
# SSL webservice daemon port
adminPort = self._portPrompt(
'Enter port for SSL webservice daemon',
self._settings['adminPort'])
# Local webservice daemon port
intWebServicePort = self._portPrompt(
'Enter port for local webservice daemon',
self._settings['intWebServicePort'])
return intWebPort, adminPort, intWebServicePort
def _removePackageSources(self):
pkgManager = self._osObjectFactory.getOsPackageManager()
for pkgSrcName in pkgManager.getPackageSourceNames():
self._logger.info(
'Removing package source [%s]' % (pkgSrcName))
pkgManager.removePackageSource(pkgSrcName)
def _disableTortugaws(self):
self.out(' * Disabling Tortuga webservice\n')
_tortugaWsManager = self._osObjectFactory.getTortugawsManager()
serviceName = _tortugaWsManager.getServiceName()
_osServiceManager = getOsObjectFactory().getOsServiceManager()
try:
_osServiceManager.stop(serviceName)
except CommandFailed:
pass
def cleanup(self):
# If possible, remove any package sources we added
self._removePackageSources()
osUtility.removeFile(self._lockFilePath)
osUtility.removeFile(self._cm.getProfileNiiFile())
# Turn off the webservice daemon
self._disableTortugaws()
# Restore resolv.conf
if osUtility.haveBackupFile('/etc/resolv.conf'):
osUtility.restoreFile('/etc/resolv.conf')
# Drop database
dbManager = self._osObjectFactory.getOsApplicationManager(
self._settings['database']['engine'])
try:
dbSchema = self._cm.getDbSchema()
self.out(' * Removing database [%s]\n' % (dbSchema))
dbManager.destroyDb(dbSchema)
except Exception as ex: # pylint: disable=broad-except
self._logger.exception(
'Could not destroy existing db: {}'.format(ex))
# Remove DB password file
osUtility.removeFile(self._cm.getDbPasswordFile())
# Remove CFM secret
cfmSecretFile = self._cm.getCfmSecretFile()
if os.path.exists(cfmSecretFile):
osUtility.removeFile(self._cm.getCfmSecretFile())
# Generic cleanup
osUtility.removeLink('/etc/tortuga-release')
# Cleanup or remove depot directory
errmsg = 'Removing contents of [%s]' % (self._settings['depotpath'])
self._logger.debug(errmsg)
if self._depotCreated:
self.out(' * %s\n' % (errmsg))
osUtility.removeDir(self._settings['depotpath'])
else:
if self._settings['depotpath']:
self.out(' * %s\n' % (errmsg))
tortugaSubprocess.executeCommand(
'rm -rf %s/*' % (self._settings['depotpath']))
self.out('\n')
if not self._forceCleaning:
self.out('Consult log(s) for further details.\n')
self._logger.error('Installation failed')
def runSetup(self):
""" Installer setup. """
self.checkPreInstallConfiguration()
# Do not run cleanup if this fails.
self.startSetup()
try:
self.preInstallPrep()
self.getClusterConfig()
self.prepDepot()
self.preConfig()
self.pre_init_db()
self.puppetBootstrap()
dbm, session = self.initDatabase()
try:
self.createAdminUser(
session,
self._settings['adminUsername'],
self._settings['adminPassword'])
self.installKits(dbm)
self.enableComponents(session)
finally:
dbm.closeSession()
self.puppetApply()
self.out('\nTortuga installation completed successfully!\n\n')
print('Run \"exec -l $SHELL\" to initialize Tortuga environment\n')
except Exception: # pylint: disable=broad-except
self._logger.exception('Fatal error occurred during setup')
raise TortugaException('Installation failed')
def _generate_db_password(self):
"""
Generate a database password.
"""
#
# Because Apache httpd server is not installed at the time this
# runs, we cannot set the ownership of this file to be 'apache'
# (which is necessary for the Tortuga webservice).
#
# Set ownership of file to root:puppet.
#
# When the Puppet bootstrap runs, it changes the ownership to
# 'apache:puppet' and everybody is happy!
#
puppet_user = pwd.getpwnam('puppet')
gid = puppet_user[3]
self._generate_password_file(self._cm.getDbPasswordFile(), gid=gid)
def _generate_redis_password(self):
"""
Generate a password for Redis.
"""
#
# Puppet needs read access to this file so that it can use it for
# writing the redis config file.
#
puppet_user = pwd.getpwnam('puppet')
gid = puppet_user[3]
self._generate_password_file(self._cm.getRedisPasswordFile(), gid=gid)
def _generate_password_file(self, file_name: str,
password_length: int = 32,
uid: int = 0, gid: int = 0,
mode: int = 0o440):
"""
Generate a password in a file.
:param file_name: the name of the file in which the password
will be stored
:param password_length: the length of the password, default = 32
:param uid: the uid (owner) of the file, default = 0
:param gid: the gid (group) of the file, default = 0
:param mode: the file perms, default 0440
"""
password = self._generate_password(password_length)
with open(file_name, 'w') as fp:
fp.write(password)
os.chown(file_name, uid, gid)
os.chmod(file_name, mode)
def _generate_password(self, length: int = 8) -> str:
"""
Generate a password.
:param length: the length of the password
:return: the generated password
"""
chars = string.ascii_letters + string.digits
return ''.join([random.choice(chars) for _ in range(length)])
def preConfig(self):
# Create default hieradata directory
hieraDataDir = '/etc/puppetlabs/code/environments/production/data'
if not os.path.exists(hieraDataDir):
os.makedirs(hieraDataDir)
# Derive host name of puppet master from FQDN
fqdn = self._settings['fqdn']
configDict = {
'version': 5,
'DNSZone': 'private',
'puppet_server': fqdn,
'depot': self._settings['depotpath'],
}
with open(os.path.join(hieraDataDir, 'tortuga-common.yaml'),
'wb') as fp:
fp.write(
yaml.safe_dump(
configDict, explicit_start=True,
default_flow_style=False).encode())
self._generate_db_password()
self._generate_redis_password()
def pre_init_db(self):
# If using 'mysql' as the database backend, we need to install the
# puppetlabs-mysql Puppet module prior to bootstrapping. This used
# to be done in 'install-tortuga.sh'
if self._settings['database']['engine'] == 'mysql':
print('\nUsing MySQL as backing database.')
puppet_module = 'puppetlabs-mysql'
logmsg = f'Installing \'{puppet_module}\' module'
self._logger.debug(logmsg)
print(f'\n{logmsg}...', end='')
cmd = ('/opt/puppetlabs/bin/puppet module install'
f' --color false {puppet_module}')
tortugaSubprocess.executeCommand(cmd)
print('done.')
def puppetBootstrap(self):
localPuppetRoot = os.path.join(self._cm.getEtcDir(), 'puppet')
logFileName = '/tmp/bootstrap.log'
puppet_server = self._settings['fqdn']
# Bootstrap using Puppet
cmd = ('/opt/puppetlabs/bin/puppet apply --verbose'
' --detailed-exitcodes'
' --execute "class { \'tortuga::installer\':'
' puppet_server => \'%s\','
'}"' % (puppet_server)
)
retval = self._runCommandWithSpinner(
cmd, '\nPerforming pre-configuration... Please wait...',
logFileName=logFileName)
if retval not in (0, 2):
# Puppet can return a non-zero return code, even if it was
# successful.
self._logger.debug(
'Puppet pre-configuration returned non-zero'
' return code [%d]' % (retval))
errmsg = 'Puppet bootstrap failed (see log file %s)' % (
logFileName)
self._logger.error(errmsg)
raise Exception(errmsg)
self._logger.debug('Puppet pre-configuration completed')
def initDatabase(self) -> Tuple[Any, Session]:
msg = _('Initializing database')
self._logger.info(msg)
print_('\n' + msg + '... ', end='')
# This cannot be a global import since the database configuration
# may be set in this script.
from tortuga.db.dbManager import DbManager
dbm = DbManager()
# create database
dbm.init_database()
session = dbm.openSession()
# Prime the database previously created as part of the bootstrap
try:
dbUtility.primeDb(session, self._settings)
dbUtility.init_global_parameters(session, self._settings)
print_(_('done'))
session.commit()
except Exception as exc: # pylint: disable=broad-except
session.rollback()
print_(_('failed.'))
print_(_('Exception raised initializing database:') +
' {0}'.format(exc), file=sys.stderr)
self._logger.debug('Done initializing database')
return dbm, session
def installKits(self, dbm):
self._logger.info('Installing kits')
self.out('\n' + _('Installing kits') + '...\n')
kitApi = KitApi()
# Iterate over the glob of 'kits-*.tar.bz2'
kitFileGlob = '%s/kits/kit-*.tar.bz2' % (self._cm.getRoot())
# Split comma-separated list of kits to skip installing. Sorry, you
# cannot skip installing the base kit.
val = self._settings['skip_kits'] \
if 'skip_kits' in self._settings else ''
skip_kits = set([
item for item in val.split(',') if item != 'base']) \
if val else set()
for kitPackage in glob.glob(kitFileGlob):
try:
kit = get_metadata_from_archive(kitPackage)
except KitNotFound:
msg = 'Kit [%s] is malformed/invalid. Skipping.' % (
os.path.basename(kitPackage))
self._logger.error(msg)
self.out(' %s\n' % (msg))
continue
if kit['name'] in skip_kits:
msg = 'Kit [%s] installation skipped.' % (kit['name'])
self.out(' %s\n' % (msg))
self._logger.info(msg)
continue
try:
kitApi.installKitPackage(dbm, kitPackage)
except EulaAcceptanceRequired:
msg = 'Kit [%s] requires EULA acceptance. Skipping.' % (
kitPackage)
self.out(' %s\n' % (msg))
self._logger.info(msg)
continue
self.out(' - %s installed.\n' % (kit['name']))
self._logger.info('Kit [%s] installed' % (kit['name']))
self._logger.info('Done installing kits')
load_kits()
def enableComponents(self, session: Session):
"""
Raises:
ConfigurationError
"""
self._logger.info('Enabling \'installer\' component')
base_kit = KitApi().getKit(session, 'base')
enabledComponents = ['installer']
# get list of components from 'base' kit
components = [c for c in base_kit.getComponentList()
if c.getName() in enabledComponents]
installerNode = NodeApi().getInstallerNode(session)
for component in components:
SoftwareProfileApi().enableComponent(
session,
installerNode.getSoftwareProfile().getName(),
base_kit.getName(),
base_kit.getVersion(),
base_kit.getIteration(),
component.getName(), compVersion=component.getVersion(),
)
def promptForAdminCredentials(self):
# Get admin username and password for use with web service
if self._settings['defaults']:
self.out(_('\nUsing default Tortuga admin user name/password.\n'))
return 'admin', 'password'
username = password = None
# Administrator username
while True:
username = self.prompt(
'admin', 'admin',
['Enter name for Tortuga admin user.',
'This user is not associated with any system user.'],
'Admin user name')
if len(username) > 3:
break
self.out('Admin user name must be at least 4 characters.')
# Administrator password
while True:
password = self.prompt(
'', 'password',
['Enter password for Tortuga admin user.'],
'Admin password', None, None, True)
if len(password) < 4:
self.out('Admin password must be at least 4 characters.')
continue
confirmPassword = self.prompt(
'', 'password',
['Confirm admin password.'],
'Confirm password', None, None, True)
if confirmPassword == password:
self.out('\n')
break
self.out('Passwords did not match.')
return username, password
def createAdminUser(self, session: Session, username, password):
msg = _('Adding administrative user')
self._logger.info(msg)
self.out('\n' + msg + '... ')
AdminApi().addAdmin(
session,
username, password, False,
description='Added by tortuga-setup')
self.out(_('done.') + '\n')
class KitManager(TortugaObjectManager):
def __init__(self, eula_validator=None):
super(KitManager, self).__init__()
self._eula_validator = eula_validator
if not self._eula_validator:
self._eula_validator = BaseEulaValidator()
self._kit_db_api = KitDbApi()
self._config_manager = ConfigManager()
self._kits_root = self._config_manager.getKitDir()
self._component_db_api = componentDbApi.ComponentDbApi()
self._logger = logging.getLogger(KIT_NAMESPACE)
def getKitList(self, session: Session):
"""
Get all installed kits.
"""
return self._kit_db_api.getKitList(session)
def getKit(self, session: Session, name, version=None, iteration=None):
"""
Get a single kit by name, and optionally version and/or iteration.
:param name: the kit name
:param version: the kit version
:param iteration: the kit iteration
:return: the kit instance
"""
return self._kit_db_api.getKit(session, name, version, iteration)
def getKitById(self,session: Session, id_):
"""
Get a single kit by id.
:param id_: the kit id
:return: the kit instance
"""
return self._kit_db_api.getKitById(session, id_)
def get_kit_url(self, name, version, iteration):
kit = Kit(name, version, iteration)
native_repo = repoManager.getRepo()
return os.path.join(
native_repo.getRemoteUrl(), kit.getTarBz2FileName())
def installKit(self, db_manager, name, version, iteration):
"""
Install kit using kit name/version/iteration.
The kit package must be located in the remote repository for the
native OS.
Kit will be installed for all operating systems that:
1) have repo configured on the local machine
2) are specified in the kit.xml file
Raises:
KitAlreadyExists
"""
kit = Kit(name, version, iteration)
# nativeRepo = repoManager.getRepo()
kitPkgUrl = self.get_kit_url(name, version, iteration)
# Check for kit existence.
with db_manager.session() as session:
self._check_if_kit_exists(session, kit)
self._logger.debug(
'[{0}] Installing kit [{1}]'.format(
self.__class__.__name__, kit))
return self.installKitPackage(db_manager, kitPkgUrl)
def _check_if_kit_exists(self, session: Session, kit):
"""
Check if a kit exists, if it does then raise an exception.
:raisesKitAlreadyExists:
"""
try:
self._kit_db_api.getKit(
session, kit.getName(), kit.getVersion(), kit.getIteration())
raise KitAlreadyExists(
'Kit already exists: ({}, {}, {})'.format(
kit.getName(), kit.getVersion(), kit.getIteration()
)
)
except KitNotFound:
pass
def installKitPackage(self, db_manager, kit_pkg_url):
"""
Install kit from the given kit url (url might be a local file).
:param db_manager: a database manager instance
:param kit_pkg_url: the URL to the kit package archive
:raises KitAlreadyExists:
:raises EulaAcceptanceRequired:
"""
self._logger.debug(
'Installing kit package: {}'.format(kit_pkg_url))
with db_manager.session() as session:
installer = self._prepare_installer(kit_pkg_url)
installer.session = session
kit = installer.get_kit()
try:
self._run_installer(db_manager, installer)
except Exception as ex:
self._delete_kit(session, kit, force=False)
raise ex
return kit
def _prepare_installer(self, kit_pkg_url):
"""
Extracts a kit archive and prepares the kit installer for the
installation process.
:param kit_pkg_url: the URL to the kit package archive
:return: the KitInstaller instance
"""
#
# Download/copy kit archive.
#
kit_src_path = os.path.basename(kit_pkg_url)
kit_pkg_path = utils.retrieve(
kit_src_path, kit_pkg_url, self._kits_root)
#
# Make sure the kit version is compatible
#
kit_meta = utils.get_metadata_from_archive(kit_pkg_path)
kit_spec = (kit_meta['name'], kit_meta['version'],
kit_meta['iteration'])
requires_core = kit_meta.get('requires_core', VERSION)
if not version_is_compatible(requires_core):
errmsg = 'The {} kit requires tortuga core >= {}'.format(
kit_meta['name'], requires_core)
raise OperationFailed(errmsg)
#
# Unpack the archive
#
kit_dir = utils.unpack_kit_archive(kit_pkg_path, self._kits_root)
#
# Load and initialize kit installer
#
load_kits()
try:
installer = get_kit_installer(kit_spec)()
assert installer.is_installable()
except Exception as ex:
if os.path.exists(kit_dir):
self._logger.debug(
'Removing kit installation directory: {}'.format(kit_dir))
osUtility.removeDir(kit_dir)
self._logger.warning(
'Kit is not installable: {}'.format(kit_spec))
return
return installer
def _run_installer(self, db_manager: DbManager, installer):
"""
Runs the installation process for a kit installer.
:param db_manager: the DbManager instance
:param installer: the KitInstaller instance to run the install process
for
"""
kit = installer.get_kit()
#
# This method will throw KitAlreadyExists, if it does...
#
self._check_if_kit_exists(installer.session, kit)
#
# Validate eula
#
eula = installer.get_eula()
if not eula:
self._logger.debug('No EULA acceptance required')
else:
if not self._eula_validator.validate_eula(eula):
raise EulaAcceptanceRequired(
'You must accept the EULA to install this kit')
#
# Runs the kit pre install method
#
installer.run_action('pre_install')
#
# Get list of operating systems supported by this installer
#
os_info_list = [
repo.getOsInfo() for repo in repoManager.getRepoList()
]
#
# Install operating system specific packages
#
self._install_os_packages(kit, os_info_list)
#
# Initialize any DB tables provided by the kit
#
db_manager.init_database()
#
# Add the kit to the database
#
self._kit_db_api.addKit(installer.session, kit)
#
# Clean up the kit archive directory
#
self._clean_kit_achiv_dir(kit, installer.install_path)
#
# Install puppet modules
#
installer.run_action('install_puppet_modules')
#
# Run post install
#
installer.run_action('post_install')
if eula:
ActionManager().logAction(
'Kit [{}] installed and EULA accepted at [{}]'
' local machine time.'.format(
installer.spec, time.ctime())
)
else:
ActionManager().logAction(
'Kit [{}] installed at [{}] local machine time.'.format(
installer.spec, time.ctime())
)
def _install_os_packages(self, kit, os_info_list):
"""
Installs OS specific packages from the kit into the repo.
:param kit: the kit instance
:param os_info_list: a list of osInfo instances to install for
"""
all_component_list = kit.getComponentList()
for os_info in os_info_list:
self._logger.debug(
'Preparing to install ({}, {}, {}) for {}'.format(
kit.getName(), kit.getVersion(), kit.getIteration(),
os_info
)
)
#
# Get list of compatible components
#
os_object_factory = getOsObjectFactory(
mapOsName(os_info.getName())
)
component_manager = os_object_factory.getComponentManager()
component_list = component_manager.getCompatibleComponentList(
os_info, all_component_list)
if not component_list:
continue
#
# Create the package directory in the repo
#
repo = repoManager.getRepo(os_info)
full_dir = os.path.join(repo.getLocalPath(), kit.getKitRepoDir())
osUtility.createDir(full_dir)
#
# Install the packages into the repo package directory
#
for component in component_list:
self._logger.debug(
'[{0}] Found component [{1}]'.format(
self.__class__.__name__, component))
for package in component.getPackageList():
package_file = os.path.join(
kit.install_path, package.getRelativePath())
self._logger.debug(
'[{0}] Found package [{1}]'.format(
self.__class__.__name__, package_file))
repo.addPackage(package_file, kit.getKitRepoDir())
repo.create(kit.getKitRepoDir())
def get_kit_eula(self, name, version, iteration=None):
return self.get_kit_package_eula(
self.get_kit_url(name, version, iteration)
)
def get_kit_package_eula(self, kit_pkg_url):
#
# Download/copy and unpack kit archive.
#
kit_src_path = os.path.basename(kit_pkg_url)
kit_pkg_path = utils.retrieve(
kit_src_path, kit_pkg_url, self._kits_root)
kit_spec = utils.unpack_archive(kit_pkg_path, self._kits_root)
#
# Get the EULA from the installer
#
installer = get_kit_installer(kit_spec)()
eula = installer.get_eula()
return eula
def _retrieveOSMedia(self, url: str) -> str:
"""
Download the OS media.
:param url: String
:return: String file path to download
"""
return urllib.request.urlretrieve(url)[0]
def _processMediaspec(self, os_media_urls: List[str]) -> List[dict]:
"""
:param os_media_urls: List String
:return: List Dictionary
"""
media_list: List[dict] = []
for url in os_media_urls:
m = urllib.parse.urlparse(url)
media_item_dict = {
'urlparse': m,
}
if m.scheme.lower() in ('http', 'https') and \
os.path.splitext(m.path)[1].lower() == '.iso':
file_name = self._retrieveOSMedia(m.geturl())
media_item_dict['localFilePath'] = file_name
media_list.append(media_item_dict)
return media_list
@staticmethod
def _getKitOpsClass(os_family_info) -> Any:
"""
Import the KitOps class for the specified OS family name
Raises:
OsNotSupported
"""
try:
_temp = __import__(
'tortuga.kit.%sOsKitOps' % os_family_info.getName(),
globals(),
locals(),
['KitOps'],
0
)
return getattr(_temp, 'KitOps')
except ImportError:
raise OsNotSupported('Currently unsupported distribution')
def _checkExistingKit(self, session: Session, kitname: str,
kitversion: str, kitarch: str):
"""
Raises:
KitAlreadyExists
"""
try:
kit = self._kit_db_api.getKit(session, kitname, kitversion)
longname = format_kit_descriptor(kitname, kitversion, kitarch)
# Attempt to get matching OS component
for c in kit.getComponentList(session):
if c.getName() != longname:
continue
for cOs in c.getOsInfoList():
if cOs == OsInfo(kitname, kitversion, kitarch):
raise KitAlreadyExists(
"OS kit [%s] already installed" % longname)
# Kit exists, but doesn't have a matching os component
except KitNotFound:
pass
def _create_kit_db_entry(self, session: Session, kit) -> Kit:
"""
Creates a database entry for a kit.
:param kit:
:return: a Kit instance (TortugaObject)
"""
try:
return self._kit_db_api.getKit(session, kit['name'], kit['ver'])
except KitNotFound:
pass
# Add the database entries for the kit
kitObj = Kit(name=kit['name'], version=kit['ver'], iteration='0')
kitObj.setDescription(kit['sum'])
kitObj.setIsOs(True)
kitObj.setIsRemovable(True)
kit_descr = format_kit_descriptor(kit['name'], kit['ver'], kit['arch'])
newComp = Component(name=kit_descr, version=kit['ver'])
newComp.setDescription('%s mock component' % (kit_descr))
newComp.addOsInfo(
osHelper.getOsInfo(kit['name'], kit['ver'], kit['arch']))
kitObj.addComponent(newComp)
# Kit does not previously exist, perform 'normal' add kit operation
self._kit_db_api.addKit(session, kitObj)
return kitObj
def installOsKit(self, session: Session, os_media_urls: List[str],
**kwargs) -> Kit:
"""
:param os_media_urls:
:param kwargs:
:return:
"""
media_list: List[dict] = self._processMediaspec(os_media_urls)
os_distro = None
kit_ops = None
enable_proxy = False
mount_manager = None
is_interactive = kwargs['bInteractive'] \
if 'bInteractive' in kwargs else False
use_symlinks = kwargs['bUseSymlinks'] \
if 'bUseSymlinks' in kwargs else False
# If 'mirror' is True, treat 'mediaspec' as a mirror, instead of
# specific OS version. This affects the stored OS version.
is_mirror = kwargs['mirror'] if 'mirror' in kwargs else False
media: dict = media_list[0] # For now, remove support for multiple ISOs / mirrors.
source_path = None
mount_manager_source_path = None
try:
if use_symlinks:
source_path = media['urlparse'].path
elif 'localFilePath' in media:
# Remote ISO file has been transferred locally and
# filename is stored in 'localFilePath'
mount_manager_source_path = media['localFilePath']
else:
pr = media['urlparse']
if pr.scheme.lower() in ('http', 'https'):
# This is a proxy URL
source_path = pr.geturl()
enable_proxy = True
elif not pr.scheme or pr.scheme.lower() == 'file':
if os.path.ismount(pr.path):
# Mount point specified
source_path = pr.path
else:
mount_manager_source_path = pr.path
else:
raise UnrecognizedKitMedia(
'Unhandled URL scheme [%s]' % pr.scheme)
# Mount source media, as necessary
if mount_manager_source_path:
mount_manager = MountManager(mount_manager_source_path)
mount_manager.mountMedia()
source_path = mount_manager.mountpoint
if os_distro is None:
# Determine the OS we're working with...
os_distro = DistributionFactory(source_path)()
if os_distro is None:
raise OsNotSupported('Could not match media')
os_info = os_distro.get_os_info()
# Check if OS is already installed before attempting to
# perform copy operation...
try:
self._checkExistingKit(
session,
os_info.getName(),
os_info.getVersion(),
os_info.getArch())
except KitAlreadyExists:
if mount_manager_source_path:
mount_manager.unmountMedia()
if 'localFilePath' in media:
if os.path.exists(
media['localFilePath']):
os.unlink(media['localFilePath'])
raise
kit_ops_class = self._getKitOpsClass(
os_info.getOsFamilyInfo())
kit_ops = kit_ops_class(
os_distro, bUseSymlinks=use_symlinks, mirror=is_mirror)
kit = kit_ops.prepareOSKit()
# Copy files here
if enable_proxy:
kit_ops.addProxy(source_path)
else:
descr = None
if is_interactive:
descr = "Installing..."
kit_ops.copyOsMedia(descr=descr)
finally:
if mount_manager_source_path:
# MountManager instance exists. Unmount any mounted
# path
mount_manager.unmountMedia()
if 'localFilePath' in media:
if os.path.exists(media['localFilePath']):
os.unlink(media['localFilePath'])
kit_object = self._create_kit_db_entry(session, kit)
self._postInstallOsKit(session, kit_object)
return kit_object
def _postInstallOsKit(self, session: Session, kit):
"""
Enable the OS component that may already be associated with an
existing software profile. This is possible when OS media is
not available during installation/creation of software profiles
"""
osComponents = kit.getComponentList()
kitOsInfo = osComponents[0].getOsComponentList()[0].getOsInfo()
# Load the newly added kit component from the database
c = self._component_db_api.getComponent(
session,
osComponents[0].getName(),
osComponents[0].getVersion(),
kitOsInfo
)
# Iterate over all software profiles looking for matching OS
for swProfile in \
SoftwareProfileApi().getSoftwareProfileList(session):
if swProfile.getOsInfo() != kitOsInfo:
continue
# Ensure OS component is enabled on this software profile
try:
self._component_db_api.addComponentToSoftwareProfile(
session, c.getId(), swProfile.getId())
except SoftwareProfileComponentAlreadyExists:
# Not an error...
pass
def _clean_kit_achiv_dir(self, kit, kit_dir):
"""
Remove packages from the kit archive directory
"""
component_list = kit.getComponentList()
if not component_list:
self._logger.debug('No components found')
return
for component in component_list:
self._logger.debug(
'Found component: {}'.format(component))
for package in component.getPackageList():
package_path = os.path.join(kit_dir,
package.getRelativePath())
if os.path.exists(package_path):
self._logger.debug(
'Deleting package: {}'.format(package_path))
os.remove(package_path)
else:
self._logger.debug(
'Skipping non-existent package: {}'.format(
package_path))
def deleteKit(self, session: Session, name, version=None, iteration=None,
force=False):
"""
Delete a kit.
:param session: a database session
:param name: the kit name
:param version: the kit version
:param iteration: the kit iteration
:param force: whether or not to force the deletion
"""
kit = self.getKit(session, name, version, iteration)
if kit.getIsOs():
self._delete_os_kit(session, kit, force)
else:
self._delete_kit(session, kit, force)
self._logger.info('Deleted kit: {}'.format(kit))
def _delete_os_kit(self, session: Session, kit, force):
"""
Deletes an OS kit.
:param kit: the Kit instance
:param force: whether or not to force the deletion
"""
self._cleanup_kit(session, kit, force)
def _delete_kit(self, session: Session, kit: Kit, force: bool):
"""
Deletes a regular kit.
:param session: a database instance
:param kit: the Kit instance
:param force: whether or not to force the deletion
"""
kit_spec = (kit.getName(), kit.getVersion(), kit.getIteration())
#
# If the kit does not exist in the DB, then we want to skip
# the step of removing it from the DB
#
skip_db = False
try:
self.getKit(session, *kit_spec)
except KitNotFound:
skip_db = True
kit_install_path = os.path.join(self._kits_root, kit.getDirName())
if os.path.exists(kit_install_path):
#
# Attempt to get the kit installer
#
installer = None
try:
installer = get_kit_installer(kit_spec)()
installer.session = session
except KitNotFound:
pass
#
# Attempt to run pre-uninstall action
#
if installer:
try:
installer.run_action('pre_uninstall')
except Exception as ex:
self._logger.warning(
'Error running pre_uninstall: {}'.format(
str(ex)
)
)
#
# Remove db record and files
#
self._cleanup_kit(session, kit, force, skip_db)
#
# Attempt to uninstall puppet modules, and perform post-install
#
if installer:
try:
installer.run_action('uninstall_puppet_modules')
except Exception as ex:
self._logger.warning(
'Error uninstalling puppet modules: {}'.format(
str(ex)
)
)
try:
installer.run_action('post_uninstall')
except Exception as ex:
self._logger.warning(
'Error running post-install: {}'.format(
str(ex)
)
)
def _cleanup_kit(self, session: Session, kit: Kit, force: bool,
skip_db: bool = False):
"""
Uninstalls the kit and it's file repos.
:param session: a database session
:param kit: the Kit instance
:param force: whether or not to force the deletion
"""
repo_dir = kit.getKitRepoDir()
#
# Remove the kit from the DB
#
if not skip_db:
self._kit_db_api.deleteKit(session, kit.getName(),
kit.getVersion(), kit.getIteration(),
force=force)
#
# Remove the files and repo
#
for repo in repoManager.getRepoList():
#
# Delete the repo
#
repo.delete(repo_dir)
#
# Remove repo files
#
full_repo_dir = os.path.join(repo.getLocalPath(), repo_dir)
self._logger.debug(
'Removing repo dir: {}'.format(full_repo_dir))
#
# When LINKOSKITMEDIA is used, the kit directory is a symlink
# to the real media, delete the link instead of attempting
# to delete the directory.
#
if os.path.islink(full_repo_dir):
os.unlink(full_repo_dir)
else:
osUtility.removeDir(full_repo_dir)
#
# Check and clean up proxy
#
self.remove_proxy(repo_dir)
#
# Remove the kit installation dir
#
kit_dir = os.path.join(self._kits_root, kit.getDirName())
if os.path.exists(kit_dir):
self._logger.debug(
'Removing kit installation directory: {}'.format(kit_dir))
osUtility.removeDir(kit_dir)
def remove_proxy(self, repoDir):
# Check for this repo as an actual entry in the tortuga apache.conf
# file to remove
config = configparser.ConfigParser()
cfgfile = os.path.join(
self._config_manager.getKitConfigBase(),
'base/apache-component.conf'
)
config.read(cfgfile)
paths = config.get('cache', 'cache_path_list').split()
newPaths = paths[:]
for p in paths:
if p.endswith(repoDir):
# Remove this dir from the cache
newPaths.remove(p)
if config.has_option('proxy', 'proxy_list'):
proxies = config.get('proxy', 'proxy_list').split()
if repoDir in ' '.join(proxies):
# Remove this repository from the proxy
for entry in proxies[:]:
if repoDir in entry:
config.remove_option('proxy', entry)
proxies.remove(entry)
else:
proxies = []
# Update apache configuration
config.set('cache', 'cache_path_list', '\n'.join(newPaths))
config.set('proxy', 'proxy_list', ' '.join(proxies))
with open(cfgfile, 'w') as fp:
config.write(fp)
def configureProxy(self, medialoc, repoDir):
config = configparser.ConfigParser()
cfgfile = os.path.join(
self._config_manager.getKitConfigBase(), 'base/apache-component.conf')
config.read(cfgfile)
proxies = []
paths = []
if config.has_section('cache') and \
config.has_option('cache', 'cache_path_list'):
paths = config.get('cache', 'cache_path_list').split()
if repoDir not in paths:
paths.append(repoDir)
if not config.has_section('cache'):
config.add_section('cache')
config.set('cache', 'cache_path_list', '\n'.join(paths))
if config.has_section('proxy') and \
config.has_option('proxy', 'proxy_list'):
proxies = config.get('proxy', 'proxy_list').split()
if repoDir not in proxies:
proxies.append(repoDir)
if not config.has_section('proxy'):
config.add_section('proxy')
config.set('proxy', 'proxy_list', ' '.join(proxies))
config.set('proxy', repoDir, medialoc)
with open(cfgfile, 'w') as fp:
config.write(fp)
class ComponentInstaller(ComponentInstallerBase):
"""
Tortuga DHCP component.
"""
name = 'dhcpd'
version = '7.0.3'
os_list = [
{'family': 'rhel', 'version': '6', 'arch': 'x86_64'},
{'family': 'rhel', 'version': '7', 'arch': 'x86_64'},
]
installer_only = True
def __init__(self, kit):
"""
Initialise parent class.
"""
super().__init__(kit)
self._provider = DhcpdDhcpProvider(self)
self._manager = self._get_os_dhcpd_manager('dhcpd')
self._config = ConfigManager()
def _get_os_dhcpd_manager(self, name):
"""
Get dhcpd manager for the appropriate os.
:param name: the name of the dhcpd manager to get
:returns: the dhcpd manager instance
"""
dir_name = '{}/util'.format(self.kit_installer.kit_path)
dhcpd_manager = \
getOsObjectFactory().getOsKitApplicationManager(name, dir_name)
return dhcpd_manager
def _get_provisioning_networks(self):
"""
Get provisioning networks.
:returns: Generator provisioning networks
"""
for network in NetworksDbHandler().getNetworkList(self.session):
if network.type == 'provision':
yield network
def _get_provisioning_nics(self, node):
"""
Get provisioning nics.
:param node: Node object
:returns: Generator nics
"""
for nic in node.getNics():
if nic.getNetwork().getType() == 'provision':
yield nic
def _get_provisioning_nics_ip(self, node):
"""
Get provisioning nics IP addresses.
:param node: Node object
:returns: Generator IPv4Address
"""
for nic in self._get_provisioning_nics(node):
yield ipaddress.IPv4Address(nic.getIp())
@staticmethod
def _get_local_nics(nics):
"""
Get valid NICs.
:returns: Generator nics
"""
for nic in nics:
if nic.boot and nic.mac:
yield nic
def _get_installer_ip(self, network_id):
"""
Return IP address of provisioning interface on installer
:raises NicNotFound:
"""
installer_node = NodeApi().getInstallerNode(self.session)
prov_nics = self._get_provisioning_nics(installer_node)
for prov_nic in prov_nics:
if prov_nic.getNetwork().getId() == network_id:
return ipaddress.IPv4Address(prov_nic.getIp())
raise NicNotFound(
'Network has no corresponding provisioning NIC on installer')
def _dhcp_subnets(self):
"""
DHCP subnet dictionary.
:returns: Dictionary IPv4Network network address IPv4Network subnet
"""
subnets = {}
for network in self._get_provisioning_networks():
subnet = {'nodes': []}
installer_ip = self._get_installer_ip(network.id)
subnet['installerIp'] = installer_ip
if not network.gateway:
logger.info(
'[dhcpd] Gateway not defined for network [{}/{}], using'
' IP [{}]'.format(
network.address,
network.netmask,
installer_ip
)
)
subnet['gateway'] = installer_ip
else:
subnet['gateway'] = network.gateway
for nic in self._get_local_nics(network.nics):
node = nic.node
if node.hardwareprofile.location != 'local' \
or node.state == 'Deleted' \
or node.name == self._config.getInstaller():
continue
node = {
'ip': nic.ip,
'mac': nic.mac,
'fqdn': node.name,
'hostname': node.name.split('.', 1)[0],
'unmanaged': False
}
subnet['nodes'].append(node)
subnet_address = ipaddress.IPv4Network('{}/{}'.format(
network.address,
network.netmask
))
subnets[subnet_address] = subnet
return subnets
@property
def _get_kit_settings_dictionary(self):
"""
:returns: Dictionary
"""
settings = {}
config = configparser.ConfigParser()
config.read(os.path.join(
self._config.getKitConfigBase(),
'tortuga.ini'
))
if config.has_section('tortuga_kit_base'):
if config.has_option('tortuga_kit_base', 'disable_services'):
settings['disable_services'] = \
config.get('tortuga_kit_base', 'disable_services') \
.split(' ')
return settings
def _configure(self, softwareProfileName, fd, *args, **kwargs):
"""
Shim for unused arguments.
:param softwareProfileName:
:param fd:
:param *args:
:param **kwargs:
:returns: None
"""
self.action_configure(softwareProfileName, *args, **kwargs)
def action_configure(self, _, *args, **kwargs):
"""
Configure.
:param _: Unused
:param *args: Unused
:param **kwargs: Unused
:returns: None
"""
try:
result = GlobalParameterDbApi().getParameter(
self.session,
'DHCPLeaseTime'
)
dhcp_lease_time = int(result.getValue())
except ParameterNotFound:
dhcp_lease_time = 2400
try:
result = GlobalParameterDbApi().getParameter(
self.session,
'DNSZone')
dns_zone = result.getValue()
except ParameterNotFound:
dns_zone = ''
installer_node = NodeApi().getInstallerNode(self.session)
self._manager.configure(
dhcp_lease_time,
dns_zone,
self._get_provisioning_nics_ip(installer_node),
self._dhcp_subnets(),
installerNode=installer_node,
bUpdateSysconfig=kwargs.get('bUpdateSysconfig', True),
kit_settings=self._get_kit_settings_dictionary
)
def action_post_install(self, *args, **kwargs):
"""
Triggered post install.
:param *args: List Objects
:param **kwargs: Dictionary Objects
:returns: None
"""
self._provider.write()
self.action_configure(None, args, kwargs, bUpdateSysconfig=True)
def action_add_host(self, hardware_profile_name, software_profile_name,
nodes, *args, **kwargs):
"""
Triggerd at add host.
:returns: None
"""
self.action_configure(
software_profile_name,
None,
args,
kwargs,
bUpdateSysconfig=False
)
def action_delete_host(self, hardware_profile_name, software_profile_name,
nodes, *args, **kwargs):
"""
Triggered delete host.
:returns: None
"""
self.action_configure(
software_profile_name,
None,
args,
kwargs,
bUpdateSysconfig=False
)
class DbManager(TortugaObjectManager):
"""
Class for db management.
:param engine: a SQLAlchemy database engine instance
:param init: a flag that is set when the database has not yet
been initialized. If this flag is set, not attempts
will be made to load/map kit tables. This flag is
cleared once the database has been initialized.
"""
def __init__(self, engine=None):
super().__init__()
if not engine:
self._cm = ConfigManager()
self._dbConfig = self._refreshDbConfig()
engineURI = self.__getDbEngineURI()
if self._dbConfig['engine'] == 'sqlite' and \
not os.path.exists(self._dbConfig['path']):
# Ensure SQLite database file is created with proper permissions
fd = os.open(
self._dbConfig['path'], os.O_CREAT, mode=0o600)
os.close(fd)
self._engine = sqlalchemy.create_engine(engineURI)
else:
self._engine = engine
self.Session = sqlalchemy.orm.scoped_session(
sqlalchemy.orm.sessionmaker(bind=self.engine))
def _register_database_tables(self):
for kit_installer_class in get_all_kit_installers():
kit_installer = kit_installer_class()
kit_installer.register_database_tables()
@property
def engine(self):
"""
SQLAlchemy Engine object property
"""
self._register_database_tables()
return self._engine
def session(self):
"""
Database session context manager
"""
return SessionContextManager(self)
def init_database(self):
#
# Create tables
#
self._register_database_tables()
try:
ModelBase.metadata.create_all(self.engine)
except Exception:
self._logger.exception('SQLAlchemy raised exception')
raise DbError('Check database settings or credentials')
@property
def metadata(self):
return self._metadata
def __getDbEngineURI(self):
dbPort = self._dbConfig['port']
dbHost = self._dbConfig['host']
engine = self._dbConfig['engine']
dbUser = self._dbConfig['username']
dbPassword = self._dbConfig['password']
if engine == 'sqlite':
engineURI = 'sqlite:///%s' % (self._dbConfig['path'])
else:
if dbUser is not None:
if dbPassword is not None:
userspec = '%s:%s' % (dbUser, dbPassword)
else:
userspec = dbUser
else:
userspec = None
if dbPort is not None:
hostspec = '%s:%s' % (dbHost, dbPort)
else:
hostspec = dbHost
engineURI = f'{engine}+pymysql' if engine == 'mysql' else engine
engineURI += '://'
if userspec is not None:
engineURI += f'{userspec}@'
engineURI += f'{hostspec}' + '/{}'.format(self._cm.getDbSchema())
return engineURI
def _getDefaultDbEngine(self): \
# pylint: disable=no-self-use
return 'sqlite'
def _getDefaultDbHost(self): \
# pylint: disable=no-self-use
return 'localhost'
def _getDefaultDbPort(self, engine): \
# pylint: disable=no-self-use
# MySQL default port
if engine == 'mysql':
return 3306
return None
def _getDefaultDbUserName(self):
return self._cm.getDbUser()
def _getDefaultDbPassword(self):
if os.path.exists(self._cm.getDbPasswordFile()):
with open(self._cm.getDbPasswordFile()) as fp:
dbPassword = fp.read()
else:
dbPassword = None
return dbPassword
def _refreshDbConfig(self, cfg=None):
dbConfig = {}
if cfg is None:
cfg = configparser.ConfigParser()
cfg.read(os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini'))
# Database engine
val = cfg.get('database', 'engine').strip().lower() \
if cfg.has_option('database', 'engine') else \
self._getDefaultDbEngine()
dbConfig['engine'] = val
if dbConfig['engine'] == 'sqlite':
# If database is sqlite, read the path
dbConfig['path'] = cfg.get('database', 'path') \
if cfg.has_section('database') and \
cfg.has_option('database', 'path') else \
os.path.join(self._cm.getEtcDir(),
self._cm.getDbSchema() + '.sqlite')
# Database host
val = cfg.get('database', 'host') \
if cfg.has_option('database', 'host') else \
self._getDefaultDbHost()
dbConfig['host'] = val
# Database port
val = cfg.get('database', 'port') \
if cfg.has_option('database', 'port') else None
dbConfig['port'] = val if val else self._getDefaultDbPort(
engine=dbConfig['engine'])
# Database username
val = cfg.get('database', 'username') \
if cfg.has_option('database', 'username') \
else self._getDefaultDbUserName()
dbConfig['username'] = val
# Database password
val = cfg.get('database', 'password') \
if cfg.has_option('database', 'password') \
else self._getDefaultDbPassword()
dbConfig['password'] = val
return dbConfig
def get_backend_opts(self): \
# pylint: disable=no-self-use
return {
'mysql_engine': 'InnoDB',
}
def getMetadataTable(self, table):
return self._metadata.tables[table]
def openSession(self):
""" Open db session. """
return self.Session()
def closeSession(self):
"""Close scoped_session."""
self.Session.remove()
class KitInstallerBase(ConfigurableMixin, metaclass=KitInstallerMeta):
"""
Base class for kit installers.
"""
config_type = 'kit'
#
# The kit installation directory
#
install_path = None
#
# Metadata, loaded via the load_meta class method.
#
name = None
version = None
iteration = None
spec = (None, None, None)
meta = {}
#
# Attributes, provided by instances of this class
#
puppet_modules = []
def __init__(self):
self.config_manager = ConfigManager()
#
# Setup paths
#
self.kit_path = os.path.dirname(inspect.getfile(self.__class__))
self.puppet_modules_path = os.path.join(self.kit_path,
'puppet_modules')
self.files_path = os.path.join(self.kit_path, 'files')
#
# Initialize configuration
#
super().__init__()
#
# Load components and resource adapters
#
self._component_installers = {}
self._component_installers_loaded = False
#
# Web service controller classes
#
self._ws_controller_classes = []
self._ws_controller_classes_loaded = False
def get_config_base(self):
return self.config_manager.getKitConfigBase()
@classmethod
def load_meta(cls, meta_dict):
"""
Loads the meta data for the kit into the class.
:param meta_dict: A dict containing the metadata, as specified by
the KitMetadataSchema class.
"""
errors = KitMetadataSchema().validate(meta_dict)
if errors:
raise Exception('Kit metadata validation error: {}'.format(errors))
meta_dict = copy.deepcopy(meta_dict)
cls.name = meta_dict.pop('name')
cls.version = meta_dict.pop('version')
cls.iteration = meta_dict.pop('iteration')
cls.spec = (cls.name, cls.version, cls.iteration)
cls.meta = meta_dict
def _load_component_installers(self):
"""
Load component installers for this kit.
"""
if self._component_installers_loaded:
return
kit_pkg_name = inspect.getmodule(self).__package__
comp_pkg_name = '{}.components'.format(kit_pkg_name)
logger.debug(
'Searching for component installers in package: {}'.format(
comp_pkg_name))
#
# Look for the components sub-package
#
try:
comp_pkg = importlib.import_module(comp_pkg_name)
except ModuleNotFoundError:
logger.warning('No component installers found for kit: {}'.format(
kit_pkg_name))
return
#
# Walk the components sub-package, looking for component installers
#
for loader, name, ispkg in pkgutil.walk_packages(comp_pkg.__path__):
if not ispkg:
continue
full_pkg_path = '{}.{}'.format(comp_pkg_name, name)
try:
#
# Look for the component module in the package
#
comp_inst_mod = importlib.import_module(
'{}.component'.format(full_pkg_path))
#
# Look for the ComponentInstaller class in the module
#
if not hasattr(comp_inst_mod, 'ComponentInstaller'):
logger.warning(
'ComponentInstaller class not found: {}'.format(
full_pkg_path))
#
# Initialize the ComponentInstaller class and register
# it with the KitInstaller
#
comp_inst_class = comp_inst_mod.ComponentInstaller
comp_inst = comp_inst_class(self)
self._component_installers[comp_inst_class.name] = \
comp_inst
logger.debug('Component installer registered: {}'.format(
comp_inst.spec))
except ModuleNotFoundError:
logger.debug(
'Package not a component: {}'.format(full_pkg_path))
self._component_installers_loaded = True
def is_installable(self):
"""
Determines whether or not this kit is installable under the given
conditions/circumstances. Override this in your implementations as
necessary.
:return: True if it is installable, False otherwise.
"""
return True
def run_action(self, action_name, *args, **kwargs):
"""
Runs the specified action.
:param action_name: the name of the action to run
"""
try:
logger.debug('Calling kit action: {} with arguments {}, {}'.format(
action_name, args, kwargs))
action = getattr(self, 'action_{}'.format(action_name))
return action(*args, **kwargs)
except KeyError:
raise Exception('Unknown action: {}'.format(action_name))
def get_kit(self):
"""
Gets the Kit instance for this kit.
:return: a Kit instance
"""
kit = Kit(name=self.name,
version=self.version,
iteration=self.iteration)
kit.setDescription(self.meta.get('description', None))
for component_installer in self.get_all_component_installers():
kit.addComponent(component_installer.get_component())
return kit
def get_eula(self):
"""
Gets the EULA for this kit, if it exists.
:return: a Eula instance if there is a EULA file, otherwise None.
"""
eula = None
eula_path = os.path.join(self.install_path, EULA_FILE)
if os.path.exists(eula_path) and os.path.isfile(eula_path):
eula_fp = open(eula_path)
text = eula_fp.read()
eula_fp.close()
eula = Eula(text=text)
else:
logger.debug('EULA not found: {}'.format(eula_path))
return eula
def get_component_installer(self, component_name):
self._load_component_installers()
return self._component_installers[component_name]
def get_all_component_installers(self):
self._load_component_installers()
return [ci for ci in self._component_installers.values()]
def register_database_table_mappers(self):
"""
Register database table mappers for this kit.
"""
kit_pkg_name = inspect.getmodule(self).__package__
db_table_pkg_name = '{}.db.tables'.format(kit_pkg_name)
logger.debug(
'Searching for database table mappers in package: {}'.format(
db_table_pkg_name))
try:
importlib.import_module(db_table_pkg_name)
except ModuleNotFoundError:
logger.debug('No database table mappers found for kit: {}'.format(
self.spec))
def register_web_service_controllers(self):
"""
Register web service controllers for this kit.
"""
kit_pkg_name = inspect.getmodule(self).__package__
ws_pkg_name = '{}.web_service.controllers'.format(kit_pkg_name)
logger.debug(
'Searching for web service controllers in package: {}'.format(
ws_pkg_name))
try:
importlib.import_module(ws_pkg_name)
except ModuleNotFoundError:
logger.debug('No web service controllers found for kit: {}'.format(
self.spec))
def register_web_service_worker_actions(self):
"""
Register web service worker actions for this kit.
"""
kit_pkg_name = inspect.getmodule(self).__package__
ws_pkg_name = '{}.web_service.worker'.format(kit_pkg_name)
logger.debug(
'Searching for web service worker actions in package: {}'.format(
ws_pkg_name))
try:
importlib.import_module(ws_pkg_name)
except ModuleNotFoundError:
logger.debug(
'No web service worker actions found for kit: {}'.format(
self.spec))
def action_install_puppet_modules(self, *args, **kwargs):
#
# Prevent circular import
#
from .actions import InstallPuppetModulesAction
return InstallPuppetModulesAction(self)(*args, **kwargs)
def action_pre_install(self):
pass
def action_pre_uninstall(self):
pass
def action_post_install(self):
#
# Install required python packages from requirements.txt
#
requirements_path = os.path.join(self.kit_path, 'requirements.txt')
pip_install_requirements(self, requirements_path)
def action_post_uninstall(self):
pass
def action_uninstall_puppet_modules(self, *args, **kwargs):
#
# Prevent circular import
#
from .actions import UninstallPuppetModulesAction
return UninstallPuppetModulesAction(self)(*args, **kwargs)
class OSSupport(OsSupportBase):
def __init__(self, osFamilyInfo):
super(OSSupport, self).__init__(osFamilyInfo)
self._cm = ConfigManager()
self._globalParameterDbApi = GlobalParameterDbApi()
try:
depot_dir = \
self._globalParameterDbApi.getParameter('depot').getValue()
except ParameterNotFound:
# Fallback to legacy default
depot_dir = '/depot'
self._cm.setDepotDir(depot_dir)
def getPXEReinstallSnippet(self, ksurl, node, hardwareprofile=None,
softwareprofile=None): \
# pylint: disable=no-self-use
# General kickstart/kernel parameters
# Find the first nic marked as bootable
nics = [nic for nic in node.nics if nic.boot]
if not nics:
raise NicNotFound(
'Node [%s] does not have a bootable NIC' % (node.name))
# Choose the first one
nic = nics[0]
if hardwareprofile is None:
hardwareprofile = node.hardwareprofile
if softwareprofile is None:
softwareprofile = node.softwareprofile
# Use settings from software profile, if defined, otherwise use
# settings from hardware profile.
bootParams = getBootParameters(hardwareprofile, softwareprofile)
kernel = bootParams['kernel']
kernelParams = bootParams['kernelParams']
initrd = bootParams['initrd']
bootargs = [
]
if softwareprofile.os.family.version == '7':
# RHEL 7.x
bootargs.append('inst.ks=%s' % (ksurl))
else:
# RHEL 5.x and 6.x
bootargs.append('ks=%s' % (ksurl))
bootargs.append('ksdevice=%s' % (nic.networkdevice.name))
# Append kernel parameters, if defined.
if kernelParams:
bootargs.append(kernelParams)
result = '''\
kernel %s
append initrd=%s %s''' % (kernel, initrd, ' '.join(bootargs))
return result
def __get_kickstart_network_entry(self, dbNode, hardwareprofile, nic): \
# pylint: disable=no-self-use
bProvisioningNic = nic.network == hardwareprofile.nics[0].network
installer_private_ip = hardwareprofile.nics[0].ip
if not bProvisioningNic and not nic.network.usingDhcp and not nic.ip:
# Unconfigured public static IP network
return None
bActivate = False
# By default, all interfaces are enabled at on boot
bOnBoot = True
# Use the network device name, as specified in the hardware profile
netargs = [
'network --device %s' % (nic.networkdevice.name)
]
if bProvisioningNic:
netargs.append(
'--bootproto %s' % (
'static' if bProvisioningNic or
not nic.network.usingDhcp else 'dhcp'))
netargs.append('--ip=%s' % (nic.ip))
netargs.append('--netmask=%s' % (nic.network.netmask))
netargs.append('--nameserver=%s' % (installer_private_ip))
bActivate = True
else:
if nic.network and nic.network.usingDhcp:
netargs.append('--bootproto dhcp')
else:
netargs.append('--bootproto static')
if nic.ip:
netargs.append('--ip=%s' % (nic.ip))
netargs.append('--netmask=%s' % (nic.network.netmask))
else:
# Do not enable interface if it's not configured
netargs.append('--onboot=no')
bOnBoot = False
# Store provisioning network interface device name for
# later reference in the template
# Ensure all interfaces are activated
if bActivate:
netargs.append('--activate')
bDefaultRoute = True
if bProvisioningNic:
# This is the nic connected to the provisioning network.
if len(dbNode.nics) > 1:
# Disable the default route on the management network.
netargs.append('--nodefroute')
bDefaultRoute = False
else:
# Disable DNS for all interfaces other than the
# provisioning network
if bOnBoot:
netargs.append('--nodns')
if nic.network.gateway and bDefaultRoute:
netargs.append('--gateway %s' % (nic.network.gateway))
return ' '.join(netargs)
def __validate_node(self, node): \
# pylint: disable=no-self-use
"""
Raises:
NodeNotFound
NicNotFound
"""
if not node.name:
raise NodeNotFound('Node must have a name')
if not node.nics:
raise NicNotFound('Node [%s] has no associated nics' % (
node.name))
def __kickstart_get_timezone(self):
tz = self._globalParameterDbApi.getParameter(
'Timezone_zone').getValue()
# Ensure timezone does not contain any spaces
return tz.replace(' ', '_')
def __kickstart_get_network_section(self, node, hardwareprofile):
# Ensure nics are processed in order (ie. eth0, eth1, eth2...)
nics = node.nics
nics.sort(key=lambda nic: nic.networkdevice.name)
network_entries = []
hostname_set = False
# Iterate over nics, adding 'network' Kickstart entries for each
for nic in nics:
networkString = self.__get_kickstart_network_entry(
node, hardwareprofile, nic)
if not networkString:
continue
if not hostname_set and nic.boot and \
nic.network.type == 'provision':
networkString += ' --hostname=%s' % (node.name)
hostname_set = True
network_entries.append(networkString)
return '\n'.join(network_entries)
def __kickstart_get_repos(self, dbSwProfile, installer_private_ip):
repo_entries = []
for dbComponent in dbSwProfile.components:
dbKit = dbComponent.kit
if dbKit.isOs or dbKit.name != 'base':
# Do not add repos for OS kits
continue
kitVer = '%s-%s' % (dbKit.version, dbKit.iteration)
kitArch = 'noarch'
subpath = '%s/%s/%s' % (dbKit.name, kitVer, kitArch)
# Check if repository actually exists
if not os.path.exists(os.path.join(self._cm.getDepotDir(),
'kits',
subpath,
'repodata',
'repomd.xml')):
# Repository for specified kit is empty. Nothing to do...
continue
url = self._cm.getYumRootUrl(installer_private_ip) + \
'/' + subpath
repo_entries.append(
'repo --name %s --baseurl=%s' % (dbKit.name, url))
subpath = '3rdparty/%s/%s/%s' % (dbSwProfile.os.family.name,
dbSwProfile.os.family.version,
dbSwProfile.os.arch)
if os.path.exists(os.path.join(self._cm.getRoot(),
'repos',
subpath,
'repodata/repomd.xml')):
# Third-party repository contains packages, include it in
# Kickstart
url = '%s/%s' % (
self._cm.getYumRootUrl(installer_private_ip), subpath)
repo_entries.append(
'repo --name tortuga-third-party --baseurl=%s' % (url))
return repo_entries
def __get_kickstart_template(self, swprofile):
ksTemplate = os.path.join(
self._cm.getKitConfigBase(),
'kickstart-%s.tmpl' % (swprofile.os.family.name.encode('ascii')))
if not os.path.exists(ksTemplate):
ksTemplate = os.path.join(
self._cm.getKitConfigBase(),
'kickstart-%s.tmpl' % (swprofile.name.encode('ascii')))
if not os.path.exists(ksTemplate):
ksTemplate = os.path.join(
self._cm.getKitConfigBase(), 'kickstart.tmpl')
return ksTemplate
def __kickstart_get_partition_section(self, softwareprofile):
buf = """\
#!/bin/sh
# Determine how many drives we have
"""
# Temporary workaround for RHEL 5.7 based distros
# https://bugzilla.redhat.com/show_bug.cgi?format=multiple&id=709880
if softwareprofile.os.version == '5.7':
buf += 'set $(PYTHONPATH=/usr/lib/booty list-harddrives)\n'
else:
buf += 'set $(list-harddrives)\n'
buf += """
d1=$1
d2=$3
d3=$5
d4=$7
"""
clearpartstr = '''
cat >/tmp/partinfo << __PARTINFO__
zerombr
'''
disksToPreserve = []
# Need to get the drives to clear
clearpartstr += 'clearpart '
driveNumbers = []
for dbPartition in softwareprofile.partitions:
disk = dbPartition.device.split('.')[0]
if disk not in driveNumbers:
driveNumbers.append(disk)
if not dbPartition.preserve:
# This is a partition to clear
if len(driveNumbers) == 1:
# First drive
clearpartstr += ('--all --initlabel'
' --drives="${d%s:-nodisk}' % (
disk))
else:
clearpartstr += ',${d%s:-nodisk}' % (disk)
else:
disksToPreserve.append(disk)
clearpartstr += "--none" if not driveNumbers else '"'
clearpartstr += '\n'
for diskNum in driveNumbers:
if diskNum in disksToPreserve:
continue
buf += '''
dd if=/dev/zero of=$d%s bs=512 count=1
''' % (diskNum)
buf += clearpartstr
bootloaderLocation = "mbr"
# Now create partitions
for dbPartition in softwareprofile.partitions:
if dbPartition.bootLoader:
# Can't control the partition in anaconda...it will be on
# the drive with the boot partition
bootloaderLocation = 'partition'
buf += self._processPartition(dbPartition)
# now do the bootloader
buf += (
'bootloader --location=%s --driveorder=${d1:-nodisk}\n' % (
bootloaderLocation))
buf += '__PARTINFO__\n'
return buf
def __get_template_subst_dict(self, node, hardwareprofile,
softwareprofile):
hardwareprofile = hardwareprofile \
if hardwareprofile else node.hardwareprofile
softwareprofile = softwareprofile \
if softwareprofile else node.softwareprofile
installer_public_fqdn = socket.getfqdn()
installer_hostname = installer_public_fqdn.split('.')[0]
installer_private_ip = hardwareprofile.nics[0].ip
try:
private_domain = self._globalParameterDbApi.\
getParameter('DNSZone').getValue()
except ParameterNotFound:
private_domain = None
installer_private_fqdn = '%s%s%s' % (
installer_hostname,
get_installer_hostname_suffix(
hardwareprofile.nics[0], enable_interface_aliases=None),
'.%s' % (private_domain) if private_domain else '')
vals = node.name.split('.', 1)
domain = vals[1].lower() if len(vals) == 2 else ''
d = {
'fqdn': node.name,
'domain': domain,
'hostname': installer_hostname,
'installer_private_fqdn': installer_private_fqdn,
'installer_private_domain': private_domain,
'installer_private_ip': installer_private_ip,
'puppet_master_fqdn': installer_public_fqdn,
'installer_public_fqdn': installer_public_fqdn,
'ntpserver': installer_private_ip,
'os': softwareprofile.os.name,
'osfamily': softwareprofile.os.family.name,
'osfamilyvers': int(softwareprofile.os.family.version),
# These are deprecated and included for backwards compatibility
# only. Do not reference them in any new kickstart templates.
'primaryinstaller': installer_private_fqdn,
'puppetserver': installer_public_fqdn,
'installerip': installer_private_ip,
}
# Add entry for install package source
d['url'] = '%s/%s/%s/%s' % (
self._cm.getYumRootUrl(installer_private_fqdn),
softwareprofile.os.name,
softwareprofile.os.version,
softwareprofile.os.arch)
d['lang'] = 'en_US.UTF-8'
d['keyboard'] = 'us'
d['networkcfg'] = self.__kickstart_get_network_section(
node, hardwareprofile)
d['rootpw'] = self._generatePassword()
d['timezone'] = self.__kickstart_get_timezone()
d['includes'] = '%include /tmp/partinfo'
d['repos'] = '\n'.join(
self.__kickstart_get_repos(
softwareprofile, installer_private_fqdn))
# Retain this for backwards compatibility with legacy Kickstart
# templates
d['packages'] = '\n'.join([])
d['prescript'] = self.__kickstart_get_partition_section(
softwareprofile)
d['installer_url'] = self._cm.getInstallerUrl(installer_private_fqdn)
d['cfmstring'] = self._cm.getCfmPassword()
return d
def getKickstartFileContents(self, node, hardwareprofile,
softwareprofile):
# Perform basic sanity checking before proceeding
self.__validate_node(node)
template_subst_dict = self.__get_template_subst_dict(
node, hardwareprofile, softwareprofile)
with open(self.__get_kickstart_template(softwareprofile)) as fp:
tmpl = fp.read()
return Template(tmpl).render(template_subst_dict)
def _generatePassword(self): \
# pylint: disable=no-self-use
# Generate a random password, used when creating a Kickstart file
# for package-based node provisioning.
strlength = 8
strchars = string.ascii_letters + string.digits
rootpw = ''.join([choice(strchars) for _ in range(strlength)])
rootpw = crypt.crypt(str(rootpw), str(time.time()))
return rootpw
def __get_partition_mountpoint(self, dbPartition): \
# pylint: disable=no-self-use
if not dbPartition.mountPoint:
if dbPartition.fsType == 'swap':
mountPoint = 'swap'
else:
# Any partition that does not have a mountpoint defined
# is ignored.
return None
else:
mountPoint = dbPartition.mountPoint
return mountPoint
def _processPartition(self, dbPartition): \
# pylint: disable=no-self-use
mountPoint = dbPartition.mountPoint \
if dbPartition.mountPoint else \
self.__get_partition_mountpoint(dbPartition)
if not mountPoint:
return ''
result = ''
# All partitions must have a mount point and partition type
result = 'part %s --fstype %s' % (mountPoint, dbPartition.fsType)
# This will throw an exception if the size stored in the
# partition settings is not an integer.
if dbPartition.size:
result += ' --size=%d' % (dbPartition.size)
else:
# If partition size is not set or is zero, use '--recommended' flag
if mountPoint == 'swap':
result += ' --recommended'
disk, part = dbPartition.device.split('.')
optionsList = dbPartition.options.split(',') \
if dbPartition.options else []
if dbPartition.grow is not None:
result += ' --grow'
if dbPartition.maxSize is not None:
result += ' --maxsize %d' % (dbPartition.maxSize)
if optionsList:
# Add the fs options...
result += ' --fsoptions="%s"' % (','.join(optionsList))
result += ' --noformat --onpart=${d%s:-nodisk}%s' % (disk, part) \
if dbPartition.preserve else \
' --ondisk=${d%s:-nodisk}' % str(disk)
result += '\n'
return result
class KitInstallerBase(ConfigurableMixin, metaclass=KitInstallerMeta):
"""
Base class for kit installers.
"""
config_type = 'kit'
#
# The kit installation directory
#
install_path = None
#
# Metadata, loaded via the load_meta class method.
#
name = None
version = None
iteration = None
spec = (None, None, None)
meta = {}
#
# Loader state
#
ws_controllers_loaded = False
db_tables_loaded = False
#
# Attributes, provided by instances of this class
#
puppet_modules = []
task_modules = []
def __init__(self):
self.config_manager = ConfigManager()
#
# Setup paths
#
self.kit_path = os.path.dirname(inspect.getfile(self.__class__))
self.puppet_modules_path = os.path.join(self.kit_path,
'puppet_modules')
self.files_path = os.path.join(self.kit_path, 'files')
#
# Initialize configuration
#
super().__init__()
#
# Load components and resource adapters
#
self._component_installers = {}
self._component_installers_loaded = False
#
# Web service controller classes
#
self._ws_controller_classes = []
self.session = None
def get_config_base(self):
return self.config_manager.getKitConfigBase()
@classmethod
def load_meta(cls, meta_dict):
"""
Loads the meta data for the kit into the class.
:param meta_dict: A dict containing the metadata, as specified by
the KitMetadataSchema class.
"""
errors = KitMetadataSchema().validate(meta_dict)
if errors:
raise Exception('Kit metadata validation error: {}'.format(errors))
requires_core = meta_dict.get('requires_core', VERSION)
if not version_is_compatible(requires_core):
raise Exception('The {} kit requires tortuga core >= {}'.format(
meta_dict['name'], requires_core))
meta_dict = copy.deepcopy(meta_dict)
cls.name = meta_dict.pop('name')
cls.version = meta_dict.pop('version')
cls.iteration = meta_dict.pop('iteration')
cls.spec = (cls.name, cls.version, cls.iteration)
cls.meta = meta_dict
def _load_component_installers(self):
"""
Load component installers for this kit.
"""
if self._component_installers_loaded:
return
kit_pkg_name = inspect.getmodule(self).__package__
comp_pkg_name = '{}.components'.format(kit_pkg_name)
logger.debug('Searching for component installers in package: %s',
comp_pkg_name)
#
# Look for the components sub-package
#
try:
comp_pkg = importlib.import_module(comp_pkg_name)
except ModuleNotFoundError:
logger.warning('No component installers found for kit: %s',
kit_pkg_name)
return
#
# Walk the components sub-package, looking for component installers
#
for loader, name, ispkg in pkgutil.walk_packages(comp_pkg.__path__):
if not ispkg:
continue
full_pkg_path = '{}.{}'.format(comp_pkg_name, name)
try:
#
# Look for the component module in the package
#
comp_inst_mod = importlib.import_module(
'{}.component'.format(full_pkg_path))
#
# Look for the ComponentInstaller class in the module
#
if not hasattr(comp_inst_mod, 'ComponentInstaller'):
logger.warning('ComponentInstaller class not found: %s',
full_pkg_path)
#
# Initialize the ComponentInstaller class and register
# it with the KitInstaller
#
comp_inst_class = comp_inst_mod.ComponentInstaller
comp_inst = comp_inst_class(self)
comp_inst.session = self.session
self._component_installers[comp_inst_class.name] = \
comp_inst
logger.debug('Component installer registered: %s',
comp_inst.spec)
except ModuleNotFoundError:
logger.debug('Package not a component: %s', full_pkg_path)
self._component_installers_loaded = True
def is_installable(self):
"""
Determines whether or not this kit is installable under the given
conditions/circumstances. Override this in your implementations as
necessary.
:return: True if it is installable, False otherwise.
"""
return True
def run_action(self, action_name, *args, **kwargs):
"""
Runs the specified action.
:param action_name: the name of the action to run
"""
try:
action = getattr(self, 'action_{}'.format(action_name))
return action(*args, **kwargs)
except KeyError:
raise Exception('Unknown action: {}'.format(action_name))
def get_kit(self):
"""
Gets the Kit instance for this kit.
:return: a Kit instance
"""
kit = Kit(name=self.name,
version=self.version,
iteration=self.iteration)
kit.setDescription(self.meta.get('description', None))
for component_installer in self.get_all_component_installers():
kit.addComponent(component_installer.get_component())
return kit
def get_eula(self):
"""
Gets the EULA for this kit, if it exists.
:return: a Eula instance if there is a EULA file, otherwise None.
"""
eula = None
eula_path = os.path.join(self.install_path, EULA_FILE)
if os.path.exists(eula_path) and os.path.isfile(eula_path):
eula_fp = open(eula_path)
text = eula_fp.read()
eula_fp.close()
eula = Eula(text=text)
else:
logger.debug('EULA not found: %s', eula_path)
return eula
def get_component_installer(self, component_name: str):
self._load_component_installers()
return self._component_installers.get(component_name)
def get_all_component_installers(self):
self._load_component_installers()
return [ci for ci in self._component_installers.values()]
def register_database_tables(self):
"""
Register database table mappers for this kit.
"""
#
# If another kit of the same name already exists, and has loaded
# the database tables, then we don't need to do it a second time
#
for ki in get_all_kit_installers():
if ki.spec[0] == self.spec[0] and ki.db_tables_loaded:
self.__class__.db_tables_loaded = True
return
kit_pkg_name = inspect.getmodule(self).__package__
db_table_pkg_name = '{}.db.models'.format(kit_pkg_name)
logger.debug('Searching for database table mappers in package: %s',
db_table_pkg_name)
try:
importlib.import_module(db_table_pkg_name)
self.__class__.db_tables_loaded = True
except ModuleNotFoundError:
logger.debug('No database table mappers found for kit: %s',
self.spec)
def register_web_service_controllers(self):
"""
Register web service controllers for this kit.
"""
#
# If another kit of the same name already exists, and has loaded
# the ws controllers, then we don't need to do it a second time
#
for ki in get_all_kit_installers():
if ki.spec[0] == self.spec[0] and ki.ws_controllers_loaded:
self.__class__.ws_controllers_loaded = True
return
kit_pkg_name = inspect.getmodule(self).__package__
ws_pkg_name = '{}.web_service.controllers'.format(kit_pkg_name)
logger.debug('Searching for web service controllers in package: %s',
ws_pkg_name)
try:
importlib.import_module(ws_pkg_name)
self.__class__.ws_controllers_loaded = True
except ModuleNotFoundError:
logger.debug('No web service controllers found for kit: %s',
self.spec)
def register_event_listeners(self):
"""
Register event listeners for this kit.
"""
kit_pkg_name = inspect.getmodule(self).__package__
listener_pkg_name = '{}.events.listeners'.format(kit_pkg_name)
logger.debug('Searching for event listeners in package: %s',
listener_pkg_name)
try:
importlib.import_module(listener_pkg_name)
except ModuleNotFoundError:
logger.debug('No event listeners found for kit: %s', self.spec)
def action_install_puppet_modules(self, *args, **kwargs):
#
# Prevent circular import
#
from .actions import UninstallPuppetModulesAction, \
InstallPuppetModulesAction
#
# Do an uninstall first, just in case there is an old version of
# the module still hanging around. This should fail silently if the
# module is not installed.
#
UninstallPuppetModulesAction(self)(*args, **kwargs)
#
# Do the actual install
#
return InstallPuppetModulesAction(self)(*args, **kwargs)
def action_pre_install(self):
pass
def action_pre_uninstall(self):
pass
def action_post_install(self):
#
# Check for python packages to install
#
pkg_dir = os.path.join(self.install_path, 'python_packages')
if os.path.exists(pkg_dir):
self._update_python_repo(pkg_dir)
#
# Install required python packages from requirements.txt
#
requirements_path = os.path.join(self.kit_path, 'requirements.txt')
pip_install_requirements(requirements_path)
def _update_python_repo(self, pkg_dir: str):
"""
Updates the Tortuga Python repo with packages from the kit.
:param pkg_dir: the source directory from which the packages will
be copied
"""
#
# Copy the files from the pkg_dir to the Tortuga repo
#
whl_path = os.path.join(pkg_dir, '*.whl')
repo_path = os.path.join(self.config_manager.getTortugaIntWebRoot(),
'python-tortuga')
cmd = 'rsync -a {} {}'.format(whl_path, repo_path)
logger.debug(cmd)
executeCommand(cmd)
#
# Re-build the package index
#
dir2pi = os.path.join(self.config_manager.getBinDir(), 'dir2pi')
cmd = '{} {}'.format(dir2pi, repo_path)
logger.debug(cmd)
executeCommand(cmd)
def action_post_uninstall(self):
pass
def action_uninstall_puppet_modules(self, *args, **kwargs):
#
# Prevent circular import
#
from .actions import UninstallPuppetModulesAction
return UninstallPuppetModulesAction(self)(*args, **kwargs)
def action_get_metadata(self,
hardware_profile_name: Optional[str] = None,
software_profile_name: Optional[str] = None,
node_name: Optional[str] = None) -> dict:
pass
class ResourceAdapter(object): \
# pylint: disable=too-many-public-methods
'''
This is the base class for all resource adapters to derive from.
The default actions simply print a debug message to show that the
subclass did not implement the action.
'''
def __init__(self, addHostSession=None):
if '__adaptername__' not in self.__class__.__dict__:
raise NotImplementedError(
'Subclasses of ResourceAdapter must have __adaptername__'
' defined')
self._logger = logging.getLogger('tortuga.resourceAdapter.%s' %
(self.__adaptername__))
self._logger.addHandler(logging.NullHandler())
self.__installer_public_hostname = None
self.__installer_public_ipaddress = None
self.__private_dns_zone = None
# Initialize caches
self.__addHostApi = None
self.__nodeApi = None
self.__osObject = None
self.__sanApi = None
# Initialize abort flag (to "not" aborting)
self.__isAborted = False
self._cm = ConfigManager()
self._addHostSession = addHostSession
@property
def addHostSession(self):
return self._addHostSession
@property
def cacheCfgFilePath(self):
return os.path.join(self._cm.getRoot(), 'var',
'%s-instance.conf' % (self.__adaptername__))
@property
def cfgFileName(self):
return os.path.join(
self._cm.getKitConfigBase(),
'adapter-defaults-%s.conf' % (self.__adaptername__))
def hookAction(self, action, nodes, args=None):
# Only the 'default' resource adapter overrides the hookAction()
# method.
pass
def start(self,
addNodesRequest,
dbSession,
dbHardwareProfile,
dbSoftwareProfile=None): \
# pylint: disable=unused-argument
self.__trace(addNodesRequest, dbSession, dbHardwareProfile,
dbSoftwareProfile)
def validate_start_arguments(self, addNodesRequest, dbHardwareProfile,
dbSoftwareProfile):
self.__trace(addNodesRequest, dbHardwareProfile, dbSoftwareProfile)
def stop(self, hardwareProfileName, deviceName):
self.__trace(hardwareProfileName, deviceName)
def updateNode(self, session, node, updateNodeRequest): \
# pylint: disable=unused-argument
self.__trace(session, node, updateNodeRequest)
def suspendActiveNode(self, nodeId):
'''Change the given active node to an idle node'''
self.__trace(nodeId)
def idleActiveNode(self, nodeIds):
'''Change the given active node to an idle node'''
self.__trace(nodeIds)
def activateIdleNode(self, node, softwareProfileName,
softwareProfileChanged):
'''Change the given idle node to an active node'''
self.__trace(node, softwareProfileName, softwareProfileChanged)
def deleteNode(self, nodeIds):
'''Remove the given node (active or idle) from the system'''
self.__trace(nodeIds)
def _async_delete_nodes(self, nodes):
"""
Asynchronously delete nodes; calls "ResourceAdapter._delete_node()"
method for each deleted nodes
:param dbNodes: list of Nodes objects
:return: None
"""
greenlets = []
for node in nodes:
greenlets.append(gevent.spawn(self._delete_node, node))
# TODO: implement timeout
gevent.joinall(greenlets)
def _delete_node(self, node):
"""
Abstract method called to delete node from
"ResourceAdapter._async_delete_nodes()"
:param node: Nodes object
"""
def transferNode(self, nodeIdSoftwareProfileTuples,
newSoftwareProfileName):
'''Transfer the given idle node'''
self.__trace(nodeIdSoftwareProfileTuples, newSoftwareProfileName)
def startupNode(self, nodeIds, remainingNodeList=None, tmpBootMethod='n'): \
# pylint: disable=unused-argument
'''Start the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support starting')
def shutdownNode(self, nodes, bSoftReset=False): \
# pylint: disable=unused-argument
'''Shutdown the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support shutdown')
def rebootNode(self, nodes, bSoftReset=False): \
# pylint: disable=unused-argument
'''Reboot the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support rebooting')
def checkpointNode(self, nodeId): \
# pylint: disable=unused-argument
'''Checkpoint the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support checkpointing')
def revertNodeToCheckpoint(self, nodeId): \
# pylint: disable=unused-argument
'''Revert the given node to the checkpoint'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support checkpointing')
def migrateNode(self, nodeId, remainingNodeList, liveMigrate): \
# pylint: disable=unused-argument
'''Migrate the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support migrating')
def addVolumeToNode(self, node, volume, isDirect): \
# pylint: disable=unused-argument
'''Add a disk to a node'''
# By default raise unsupported operation
raise UnsupportedOperation(
'Node does not support dynamic disk addition')
def removeVolumeFromNode(self, node, volume): \
# pylint: disable=unused-argument
'''Remove a disk from a node'''
# By default raise unsupported operation
raise UnsupportedOperation(
'Node does not support dynamic disk deletion' % (node))
def abort(self):
'''abort node addition'''
self._logger.debug('Setting abort flag')
self.__isAborted = True
def isAborted(self):
'''Returns status of abort flag'''
return self.__isAborted
def __trace(self, *pargs, **kargs):
stack = traceback.extract_stack()
funcname = stack[-2][2]
self._logger.debug('-- (pass) %s::%s %s %s' %
(self.__adaptername__, funcname, pargs, kargs))
def getLogger(self):
return self._logger
def getResourceAdapterConfig(self, sectionName=None):
"""
Raises:
ResourceNotFound
"""
self.getLogger().debug(
'getResourceAdapterConfig(sectionName=[{0}])'.format(
sectionName if sectionName else '(none)'))
try:
# Load default values
defaultResourceAdapterConfigDict = self._loadConfigDict()
if sectionName is None or sectionName == 'default':
return defaultResourceAdapterConfigDict
except ResourceNotFound:
defaultResourceAdapterConfigDict = {}
overrideConfigDict = self._loadConfigDict(sectionName)
# Override defaults with hardware profile specific settings
return dict(
list(defaultResourceAdapterConfigDict.items()) +
list(overrideConfigDict.items()))
def _loadConfigDict(self, sectionName=None):
"""
Raises:
ResourceNotFound
"""
if sectionName is None:
sectionName = 'default'
session = DbManager().openSession()
try:
self.getLogger().debug('_loadConfigDict()')
result = ResourceAdapterCredentialsDbHandler().get(
session, self.__adaptername__, sectionName)
configDict = {}
for entry in result['configuration']:
configDict[entry['key']] = entry['value']
finally:
DbManager().closeSession()
return configDict
def getResourceAdapterConfigProfileByNodeName(self, name):
"""Get resource adapter configuration for existing node"""
self.getLogger().debug(
'getResourceAdapterConfigByNodeName(): name=[{0}]'.format(name))
instance_cache = self.instanceCacheRefresh()
return instance_cache.get(name, 'resource_adapter_configuration') \
if instance_cache.has_section(name) and instance_cache.has_option(
name, 'resource_adapter_configuration') else None
def __getAddHostApi(self):
'''Get and cache the Add Host API'''
if self.__addHostApi is None:
from tortuga.addhost.addHostServerLocal \
import AddHostServerLocal
self.__addHostApi = AddHostServerLocal()
return self.__addHostApi
def __getNodeApi(self):
'''Get and cache the Node API'''
if self.__nodeApi is None:
from tortuga.node.nodeApi import NodeApi
self.__nodeApi = NodeApi()
return self.__nodeApi
def __getOsObject(self):
'''Get and cache the OS Object Factory'''
if self.__osObject is None:
from tortuga.os_utility import osUtility
self.__osObject = osUtility.getOsObjectFactory()
return self.__osObject
def __getSanApi(self):
'''Internal: Get and cache the SAN API'''
if self.__sanApi is None:
from tortuga.san import san
self.__sanApi = san.San()
return self.__sanApi
# Properties for this object
addHostApi = property(__getAddHostApi, None, None, None)
nodeApi = property(__getNodeApi, None, None, None)
osObject = property(__getOsObject, None, None, None)
sanApi = property(__getSanApi, None, None, None)
def statusMessage(self, msg):
if self._addHostSession:
AddHostManager().updateStatus(self._addHostSession, msg)
else:
# Just print out the message...this is a stop gap for resource
# adapters running outside of the addHostManager framework
sys.stdout.write(msg + '\n')
sys.stdout.flush()
def getOptions(self, dbSoftwareProfile, dbHardwareProfile): \
# pylint: disable=unused-argument
return {}
def instanceCacheWrite(self, cfg):
# Write the instance cache back to disk
self.getLogger().debug('instanceCacheWrite()')
with open(self.cacheCfgFilePath, 'w') as fp:
cfg.write(fp)
def instanceCacheRefresh(self):
self.getLogger().debug('instanceCacheRefresh()')
cfg = configparser.ConfigParser()
cfg.read(self.cacheCfgFilePath)
return cfg
def instanceCacheSet(self, name, metadata=None):
self.getLogger().debug('instanceCacheSet(node=[%s], metadata=[%s])' %
(name, metadata))
cfg = self.instanceCacheRefresh()
if not cfg.has_section(name):
cfg.add_section(name)
# Write metadata to node section
if metadata:
for key, value in metadata.items():
cfg.set(name, key, value)
self.instanceCacheWrite(cfg)
def instanceCacheSetBulk(self, instance_ids, nodes=None):
self.getLogger().debug(
'instanceCacheSetBulk(instance_ids=[%s], nodes=[%s])' %
(' '.join(instance_ids), ' '.join(
[node.name for node in nodes or []])))
cfg = self.instanceCacheRefresh()
if not nodes:
if not cfg.has_section('unassigned'):
cfg.add_section('unassigned')
instances = set()
else:
val = cfg.get('unassigned', 'instances')
instances = set(val.split(' '))
instances |= set(instance_ids)
cfg.set('unassigned', 'instances', ' '.join(instances))
self.instanceCacheWrite(cfg)
def instanceCacheGet(self, nodeName):
self.getLogger().debug('instanceCacheGet(nodeName=[%s])' % (nodeName))
cfg = self.instanceCacheRefresh()
if not cfg.has_section(nodeName):
raise ResourceNotFound(
'No instance cache entry for [{0}]'.format(nodeName))
# Read entire section into a dict
result = {}
for key, value in cfg.items(nodeName):
result[key] = value
return result
def instanceCacheDelete(self, name):
# Clear instance from configuration
config = self.instanceCacheRefresh()
if not config.has_section(name):
self.getLogger().debug(
'Cache clear: node [{0}] not found, no action'
' taken'.format(name))
return
self.getLogger().debug('Cache clear: node [{0}]'.format(name))
config.remove_section(name)
self.instanceCacheWrite(config)
def instanceCacheUpdate(self, name, added=None, deleted=None):
"""
'added' is a list of key-value tuples to be added
'deleted' is a list of keys to be removed from the instance cache
"""
self.getLogger().debug(
'instanceCacheUpdate(): name=[{0}]'.format(name))
config = self.instanceCacheRefresh()
if not config.has_section(name):
config.add_section(name)
for key, value in added or []:
config.set(name, key, value)
for key in deleted or []:
config.remove_option(name, key)
self.instanceCacheWrite(config)
def __findNicForProvisioningNetwork(self, nics, prov_network):
"""
TODO: move this elsewhere
Raises:
NicNotFound
"""
nics = [nic for nic in nics if nic.network == prov_network]
if not nics:
raise NicNotFound(
'Unable to find NIC on provisioning network [%s]' %
(prov_network.address + '/' + prov_network.netmask))
return nics[0]
def writeLocalBootConfiguration(self, node, hardwareprofile,
softwareprofile):
"""
Raises:
NicNotFound
"""
if not hardwareprofile.nics:
# Hardware profile has no provisioning NICs defined. This
# shouldn't happen...
self.getLogger().debug(
'No provisioning nics defined in hardware profile %s' %
(hardwareprofile.name))
return
# Determine the provisioning nic for the hardware profile
hwProfileProvisioningNic = hardwareprofile.nics[0]
nic = None
if hwProfileProvisioningNic.network:
# Find the nic attached to the newly added node that is on
# the same network as the provisioning nic.
nic = self.__findNicForProvisioningNetwork(
node.nics, hwProfileProvisioningNic.network)
if not nic or not nic.mac:
self.getLogger().warning(
'MAC address not defined for nic (ip=[%s]) on node [%s]' %
(nic.ip, node.name))
return
# Set up DHCP/PXE for newly addded node
bhm = getOsObjectFactory().getOsBootHostManager()
# Write out the PXE file
bhm.writePXEFile(node,
hardwareprofile=hardwareprofile,
softwareprofile=softwareprofile,
localboot=False)
# Add a DHCP lease
bhm.addDhcpLease(node, nic)
def removeLocalBootConfiguration(self, node):
bhm = self.osObject.getOsBootHostManager()
bhm.rmPXEFile(node)
bhm.removeDhcpLease(node)
def _pre_add_host(self, name, hwprofilename, swprofilename, ip): \
# pylint: disable=unused-argument
# Perform "pre-add-host" operation
command = ('sudo %s/pre-add-host'
' --hardware-profile %s'
' --software-profile %s'
' --host-name %s' %
(self._cm.getBinDir(), hwprofilename, swprofilename, name))
if ip:
command += ' --ip %s' % (ip)
self.getLogger().debug('calling command= [%s]' % (command))
p = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
p.communicate()
p.wait()
@property
def installer_public_hostname(self):
if self.__installer_public_hostname is None:
cmd = '/opt/puppetlabs/bin/facter fqdn'
with open(os.devnull, 'w') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=devnull)
stdout, _ = p.communicate()
retval = p.wait()
if retval == 0:
self.__installer_public_hostname = stdout.decode().rstrip()
self.getLogger().debug('using installerName [%s] from Facter' %
(self.__installer_public_hostname))
else:
self.__installer_public_hostname = self._cm.getHost()
self.getLogger().debug('using installerName [%s] from system' %
(self.__installer_public_hostname))
return self.__installer_public_hostname
@property
def installer_public_ipaddress(self):
# Get installer IP
if self.__installer_public_ipaddress is None:
self.getLogger().debug('Looking up installer IP using DNS')
aiInfo = socket.getaddrinfo(self.installer_public_hostname, None,
socket.AF_INET, socket.SOCK_STREAM)
self.__installer_public_ipaddress = aiInfo[0][4][0]
return self.__installer_public_ipaddress
@property
def private_dns_zone(self):
if self.__private_dns_zone is None:
self.__private_dns_zone = \
ParameterApi().getParameter('DNSZone').getValue()
return self.__private_dns_zone
def get_node_vcpus(self, name): \
# pylint: disable=unused-argument
return 1
def get_instance_size_mapping(self, value):
"""
Helper method for matching the first field (instance size) in
the resource adapter specific CSV file
:return: instance type/size to vcpus mapping
:returntype int:
"""
fn = os.path.join(
self._cm.getKitConfigBase(),
'{0}-instance-sizes.csv'.format(self.__adaptername__))
if not os.path.exists(fn):
return 1
try:
with open(fn) as fp:
reader = csv.reader(fp)
for row in reader:
if row[0] == value:
return int(row[1])
return 1
except Exception as exc:
self.getLogger().error(
'Error processing instance type mapping'
' [{0}] (exc=[{1}]). Using default value'.format(fn, exc))
return 1
