def _install_puppet_module(self, module_name):
"""
Installs a puppet module from the kit puppet_modules directory.
:param module_name: The name of the puppet module to install.
"""
files = glob.glob(
os.path.join(self.kit_installer.puppet_modules_path,
'{}-*.tar.gz'.format(module_name)))
if not files:
logger.error('Unable to find Puppet module {}'.format(module_name))
raise ConfigurationError('Missing Puppet module for Base kit')
logger.info('Installing Puppet module {}'.format(module_name))
module_path = files[0]
if not os.path.exists(module_path):
raise ConfigurationError(
'File does not exist: {}'.format(module_path))
cmd = ('/opt/puppetlabs/bin/puppet module install --color false'
' --force {}'.format(module_path))
tortugaSubprocess.executeCommand(cmd)
def getFromDict(cls, _dict, ignore: Optional[Iterable[str]] = None):
hardwareProfile = super(HardwareProfile, cls).getFromDict(_dict)
hardwareProfile.setAdmins(
tortuga.objects.admin.Admin.getListFromDict(_dict))
hardwareProfile.setNodes(
tortuga.objects.node.Node.getListFromDict(_dict))
hardwareProfile.setProvisioningNics(
tortuga.objects.nic.Nic.getListFromDict(_dict))
resourceAdapterDict = _dict.get(
tortuga.objects.resourceAdapter.ResourceAdapter.ROOT_TAG)
if resourceAdapterDict:
hardwareProfile.setResourceAdapter(
tortuga.objects.resourceAdapter.ResourceAdapter.getFromDict(
resourceAdapterDict))
hardwareProfile.setNetworks(
tortuga.objects.network.Network.getListFromDict(_dict))
# Sanity checking... do not allow the parsed XML to build an
# invalid list of locations.
if 'locations' in _dict:
raise ConfigurationError(
'Hardware profile can have only one location defined')
return hardwareProfile
def getResourceAdapterConfig(self, sectionName=None):
"""
Get resource adapter configuration dict
Raises:
ConfigurationError
:arg sectionName: resource adapter configuration profile name
:return: dict containing resource adapter configuration
:rtype: dict
"""
configDict = super(Oracleadapter, self).getResourceAdapterConfig(
sectionName=sectionName)
if 'user_data_script_template' in configDict:
if not configDict['user_data_script_template'].startswith('/'):
fn = os.path.join(
self._cm.getKitConfigBase(),
configDict['user_data_script_template'])
else:
fn = configDict['user_data_script_template']
if not os.path.exists(fn):
raise ConfigurationError(
'User data script template [%s] does not exist' % fn
)
configDict['user_data_script_template'] = fn
return configDict
def get_config(
self,
profile: str = DEFAULT_CONFIGURATION_PROFILE_NAME
) -> Dict[str, Any]:
"""
Gets the resource adapter configuration for the specified profile.
:param str profile: the reousrce adapter profile to get
:return Dict[str, Any]: the configuration
:raises ConfigurationError:
:raises ResourceNotFound:
"""
self._logger.debug('get_config(profile={})'.format(profile))
#
# Validate the settings and dump the config with transformed
# values
#
try:
validator = self.validate_config(profile)
processed_config: Dict[str, Any] = validator.dump()
except ValidationError as ex:
raise ConfigurationError(str(ex))
#
# Perform any required additional processing on the config
#
self.process_config(processed_config)
return processed_config
def getResourceAdapterConfig(self,
sectionName: str = 'Default'
) -> Dict[str, Any]:
"""
Gets the resource adatper configuration for the specified profile.
:param str sectionName: the reousrce adapter profile to get
:return Dict[str, Any]: the configuration
:raises ConfigurationError:
:raises ResourceNotFound:
"""
self.getLogger().debug(
'getResourceAdapterConfig(sectionName=[{0}])'.format(
sectionName if sectionName else '(none)'))
#
# Validate the settings and dump the config with transformed
# values
#
try:
validator = self.validate_config(sectionName)
processed_config: Dict[str, Any] = validator.dump()
except ValidationError as ex:
raise ConfigurationError(str(ex))
#
# Perform any required additional processing on the config
#
self.process_config(processed_config)
return processed_config
def _install_puppet_module(self, module_name):
"""
Installs a puppet module from the kit puppet_modules directory.
:param module_name: The name of the puppet module to install.
"""
files = glob.glob(
os.path.join(
self.kit_installer.puppet_modules_path,
'{}-*.tar.gz'.format(module_name)
)
)
if not files:
errmsg = f'Unable to find Puppet module {module_name}'
logger.error(errmsg)
raise ConfigurationError(errmsg)
logger.info('Installing Puppet module {}'.format(module_name))
cmd = ('/opt/puppetlabs/bin/puppet module install --color false'
' {}'.format(files[0]))
tortugaSubprocess.executeCommand(cmd)
def pre_enable_wrapper(cls, softwareProfileName, *pargs, **kargs):
swprofile = SoftwareProfileDbApi().getSoftwareProfile(
softwareProfileName)
if swprofile.getType() == 'installer':
raise ConfigurationError(
'Component [{0}] can only be enabled on compute software'
' profiles'.format(cls.__component_name__))
return func(cls, softwareProfileName, *pargs, **kargs)
def __validateHostName(self, hostname: str, name_format: str) -> None:
"""
Raises:
ConfigurationError
"""
bWildcardNameFormat = (name_format == '*')
if hostname and not bWildcardNameFormat:
# Host name specified, but hardware profile does not
# allow setting the host name
raise ConfigurationError(
'Hardware profile does not allow setting host names'
' of imported nodes')
elif not hostname and bWildcardNameFormat:
# Host name not specified but hardware profile expects it
raise ConfigurationError(
'Hardware profile requires host names to be set')
def get_node_resource_adapter_config(self, node: Node) \
-> Dict[str, Any]:
"""
Deserialize resource adapter configuration to key/value pairs
"""
#
# Default settings dict is blank
#
config: Dict[str, str] = {}
#
# Load settings from class settings definitions if any of them
# have default values
#
config.update(self._load_config_from_class())
#
# Load settings from default profile in database, if it exists
#
config.update(self._load_config_from_database())
#
# Load node specific settings
#
override_config: Dict[str, Any] = {}
if node.instance and \
node.instance.resource_adapter_configuration and \
node.instance.resource_adapter_configuration.name != 'Default':
for c in node.instance.resource_adapter_configuration.configuration:
override_config[c.key] = c.value
config.update(override_config)
#
# Dump the config with transformed values. Don't bother validating
# here, as we will consider the node already exists in it's current
# state
#
try:
validator = ConfigurationValidator(self.settings)
validator.load(config)
processed_config: Dict[str, Any] = validator.dump()
except ValidationError as ex:
raise ConfigurationError(str(ex))
#
# Perform any required additional processing on the config
#
self.process_config(processed_config)
return processed_config
def get_ec2_region(aws_access_key_id, aws_secret_access_key, region=None):
regions = boto.ec2.regions(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
if not region:
# Use first region in list ('us-east-1')
return regions[0]
for tmpregion in regions:
if str(tmpregion.name) == region:
return tmpregion
raise ConfigurationError('Unknown AWS region: {0}'.format(region))
def is_enableable(self, software_profile):
"""
Determines whether this component can be enabled under the
given conditions/circumstances. Override this in your
implementations as necessary.
:param software_profile: the software profile to test
:return: True if it can be enabled, False otherwise.
"""
node_type = software_profile.getType()
if node_type != 'installer' and self.installer_only:
raise ConfigurationError(
'Component can only be enabled on installer software'
'profiles: {}'.format(self.name))
if node_type == 'installer' and self.compute_only:
raise ConfigurationError(
'Component can only be enabled on compute software'
'profiles: {}'.format(self.name))
return True
def __getBaseKit(self): \
# pylint: disable=no-self-use
kitApi = KitApi()
k = None
for k in kitApi.getKitList():
if k.getName() == 'base':
break
else:
raise ConfigurationError(
'Unable to find \"base\" kit. Check Tortuga installation')
return k
def _get_config_file_path(self, filepath):
"""
Raises:
ConfigurationError
"""
if filepath.startswith('/'):
fn = filepath
else:
fn = os.path.join(self._cm.getKitConfigBase(), filepath)
if not os.path.exists(fn):
raise ConfigurationError(
'Configuration file [{0}] does not exist'.format(fn))
return fn
def installPuppetModule(self, modulePath):
"""
Install "standard" Puppet module using "puppet module install --force"
Raises:
ConfigurationError
"""
if not os.path.exists(modulePath):
errmsg = ('Error: unable to install puppet module [%s].'
' Module does not exist' % (modulePath))
self.getLogger().error(errmsg)
raise ConfigurationError(errmsg)
cmd = ('/opt/puppetlabs/bin/puppet module install --color false'
' --force %s' % (modulePath))
tortugaSubprocess.executeCommand(cmd)
def get_cluster_by_swprofilename(self, name: str) -> dict:
"""
Raises:
ConfigurationError
"""
# check if UGE is installed
Uge_kit_installer = self.__get_uge_kit()
if Uge_kit_installer is None:
self._logger.info('Cannot get uge kit installer')
return
uge_mgmt_api = Uge_kit_installer.get_uge_mgmt_api()
clusters = uge_mgmt_api.get(self.session, swprofilename=name)
self._logger.info('clusters: {}'.format(clusters))
if not clusters:
raise ConfigurationError(
'Software profile [{}] is not part of a UGE cluster'.format(
name))
return clusters
def getProvisioningNetwork(self):
'''
A hardware profile may have multiple networks that are marked as
"provisioning". However, that really means "private", and only
one of them is used for actual provisioning. This figures out
which one.
'''
best = None
for network in self.getNetworks():
if network.isProvisioning():
if best is None or network.getNetworkDevice().getName() < \
best.getNetworkDevice().getName():
best = network
if best is None:
raise ConfigurationError("Can't getProvisioningNetwork for ng %s" %
(self.getName()))
return best
def expand_cloud_init_user_data_template(
self, configDict: dict,
node: Optional[Node] = None,
template=None,
insertnode_request: Optional[bytes] = None) -> str:
"""
Return cloud-init script template
Raises:
ConfigurationError
"""
if 'cloud_init_script_template' not in configDict:
raise ConfigurationError('cloud-init script template not defined')
if template is None:
srcpath, srcfile = os.path.split(
configDict['cloud_init_script_template'])
env = Environment(loader=FileSystemLoader(srcpath))
template_ = env.get_template(srcfile)
else:
template_ = template
tmpl_vars = {
'installer': self.installer_public_hostname,
'installer_ip_address': self.installer_public_ipaddress,
'override_dns_domain': configDict.get('override_dns_domain',
False),
'dns_domain': configDict.get('dns_domain', ''),
}
if insertnode_request:
tmpl_vars['insertnode_request'] = insertnode_request.decode('utf-8')
if node:
tmpl_vars['fqdn'] = node.name
return template_.render(tmpl_vars)
def __populateHardwareProfile(self,
session,
hardwareProfile,
dbHardwareProfile=None):
"""
Helper function for creating / updating HardwareProfiles. If
'dbHardwareProfile' is specified, this is an update (vs. add) operation
Raises:
NicNotFound
"""
# Preload provisioning nics and networks
prov_nics = self.__get_provisioning_nics(session)
all_networks = self.__get_all_networks(session)
networkdevices = self.__get_network_devices(session)
# Validate hw profile
if hardwareProfile.getName() is None:
raise ConfigurationError('Hardware profile requires name.')
if hardwareProfile.getNameFormat() is None:
raise ConfigurationError(
'Hardware profile requires name format field.')
# Handle the special case of a hardware profile not having an
# associated idle software profile (ie. Installer hardware
# profile)
idleSoftwareProfileId = hardwareProfile.getIdleSoftwareProfileId() \
if hardwareProfile.getIdleSoftwareProfileId else None
if dbHardwareProfile is None:
dbHardwareProfile = HardwareProfiles()
dbHardwareProfile.name = hardwareProfile.getName()
dbHardwareProfile.description = hardwareProfile.getDescription()
dbHardwareProfile.nameFormat = hardwareProfile.getNameFormat()
if hardwareProfile.getInstallType() is None:
if hardwareProfile.getLocation() == 'remote':
dbHardwareProfile.installType = 'bootstrap'
else:
raise ConfigurationError(
'Hardware profile must have valid install type.')
else:
dbHardwareProfile.installType = hardwareProfile.\
getInstallType()
if hardwareProfile.getLocation() != 'remote':
dbHardwareProfile.kernel = hardwareProfile.getKernel()
dbHardwareProfile.kernelParams = hardwareProfile.\
getKernelParams()
dbHardwareProfile.initrd = hardwareProfile.getInitrd()
dbHardwareProfile.localBootParams = hardwareProfile.\
getLocalBootParams()
dbHardwareProfile.softwareOverrideAllowed = hardwareProfile.\
getSoftwareOverrideAllowed()
dbHardwareProfile.idleSoftwareProfileId = idleSoftwareProfileId
dbHardwareProfile.location = hardwareProfile.getLocation()
dbHardwareProfile.hypervisorSoftwareProfileId = hardwareProfile.\
getHypervisorSoftwareProfileId()
dbHardwareProfile.maxUnits = hardwareProfile.getMaxUnits()
dbHardwareProfile.bcastEnabled = hardwareProfile.getBcastEnabled()
dbHardwareProfile.mcastEnabled = hardwareProfile.getMcastEnabled()
dbHardwareProfile.cost = hardwareProfile.getCost()
# Add resource adapter
resourceAdapter = hardwareProfile.getResourceAdapter()
if resourceAdapter:
dbHardwareProfile.resourceAdapter = self.\
_resourceAdaptersDbHandler.getResourceAdapter(
session, resourceAdapter.getName())
dbHardwareProfile.resourceAdapterId = dbHardwareProfile.\
resourceAdapter.id
# Add networks
networks = []
for network in hardwareProfile.getNetworks():
for prov_network in all_networks:
if prov_network.address == network.getAddress():
dbNetwork = prov_network
break
else:
raise NetworkNotFound('Network [%s] does not exist' %
(network.getAddress()))
# Lookup network device
for network_device in networkdevices:
if network.getNetworkDevice().getName() == network_device.name:
dbNetworkDevice = network_device
break
else:
dbNetworkDevice = NetworkDevices()
dbNetworkDevice.name = network.getNetworkDevice().getName()
# Now check if we have this one already...
for dbHardwareProfileNetwork in dbHardwareProfile.\
hardwareprofilenetworks:
if dbHardwareProfileNetwork.\
networkDeviceId == dbNetworkDevice.id and \
dbHardwareProfileNetwork.networkId == dbNetwork.id:
break
else:
dbHardwareProfileNetwork = HardwareProfileNetworks()
if dbNetwork.id is not None:
dbHardwareProfileNetwork.networkId = dbNetwork.id
else:
dbHardwareProfileNetwork.network = dbNetwork
dbHardwareProfileNetwork.hardwareProfileId = \
dbHardwareProfile.id
if dbNetworkDevice.id is not None:
dbHardwareProfileNetwork.networkDeviceId = \
dbNetworkDevice.id
else:
dbHardwareProfileNetwork.networkdevice = dbNetworkDevice
dbHardwareProfile.hardwareprofilenetworks.append(
dbHardwareProfileNetwork)
networks.append(dbHardwareProfileNetwork)
# Now remove all old networks
for dbNetwork in dbHardwareProfile.hardwareprofilenetworks:
for network in networks:
if network.networkDeviceId == dbNetwork.networkDeviceId \
and network.networkId == dbNetwork.networkId:
# Its a keeper
break
else:
# No match...delete time
session.delete(dbNetwork)
# Add provisioning Nics
if hardwareProfile.getProvisioningNics():
# Only one provisioning nic is possible
nic = hardwareProfile.getProvisioningNics()[0]
for prov_nic in prov_nics:
if nic.getIp() == prov_nic.ip:
dbNic = prov_nic
break
else:
raise NicNotFound('Provisioning NIC with IP [%s] not found' %
(nic.getIp()))
if dbNic not in dbHardwareProfile.nics:
dbHardwareProfile.nics.append(dbNic)
return dbHardwareProfile
def __populateHardwareProfile(
self, session: Session, hardwareProfile: HardwareProfile,
dbHardwareProfile: Optional[HardwareProfileModel] = None) \
-> HardwareProfileModel:
"""
Helper function for creating / updating hardware profiles. If
'dbHardwareProfile' is specified, this is an update (vs. add)
operation
Raises:
NicNotFound
ResourceAdapterNotFound
InvalidArgument
ConfigurationError
"""
# Preload provisioning nics and networks
prov_nics = self.__get_provisioning_nics(session)
all_networks = self.__get_all_networks(session)
# Validate hw profile
if hardwareProfile.getName() is None:
raise ConfigurationError('Hardware profile requires name.')
if hardwareProfile.getNameFormat() is None:
raise ConfigurationError(
'Hardware profile requires name format field.')
if dbHardwareProfile is None:
dbHardwareProfile = HardwareProfileModel()
dbHardwareProfile.name = hardwareProfile.getName()
dbHardwareProfile.description = hardwareProfile.getDescription()
dbHardwareProfile.nameFormat = hardwareProfile.getNameFormat()
if hardwareProfile.getInstallType() is None:
if hardwareProfile.getLocation() == 'remote':
dbHardwareProfile.installType = 'bootstrap'
else:
dbHardwareProfile.installType = 'package'
else:
dbHardwareProfile.installType = hardwareProfile.\
getInstallType()
if hardwareProfile.getLocation() != 'remote':
dbHardwareProfile.kernel = hardwareProfile.getKernel()
dbHardwareProfile.kernelParams = \
hardwareProfile.getKernelParams()
dbHardwareProfile.initrd = hardwareProfile.getInitrd()
dbHardwareProfile.localBootParams = \
hardwareProfile.getLocalBootParams()
dbHardwareProfile.softwareOverrideAllowed = hardwareProfile.\
getSoftwareOverrideAllowed()
dbHardwareProfile.location = hardwareProfile.getLocation()
dbHardwareProfile.cost = hardwareProfile.getCost()
# Add resource adapter
resource_adapter_name = \
hardwareProfile.getResourceAdapter().getName() \
if hardwareProfile.getResourceAdapter() else 'default'
dbHardwareProfile.resourceadapter = \
self._resourceAdaptersDbHandler.getResourceAdapter(
session, resource_adapter_name)
if hardwareProfile.getDefaultResourceAdapterConfig():
adapter_cfg = None
self._logger.debug(
'Setting default resource adapter config: {}'.format(
hardwareProfile.getDefaultResourceAdapterConfig())
)
for adapter_cfg in \
dbHardwareProfile.resourceadapter.resource_adapter_config:
if adapter_cfg.name == \
hardwareProfile.getDefaultResourceAdapterConfig():
break
else:
raise InvalidArgument(
'Resource adapter configuration profile [{}] is'
' invalid'.format(
hardwareProfile.getDefaultResourceAdapterConfig())
)
dbHardwareProfile.default_resource_adapter_config = adapter_cfg
else:
dbHardwareProfile.default_resource_adapter_config = None
# Add networks
networks = []
for network in hardwareProfile.getNetworks():
for prov_network in all_networks:
if prov_network.address == network.getAddress():
dbNetwork = prov_network
break
else:
raise NetworkNotFound(
'Network [%s] does not exist' % (network.getAddress()))
dbNetworkDevice = \
self._networkDevicesDbHandler.createNetworkDeviceIfNotExists(
session, network.getNetworkDevice().getName())
# Now check if we have this one already...
for dbHardwareProfileNetwork in \
dbHardwareProfile.hardwareprofilenetworks:
if dbHardwareProfileNetwork.networkDeviceId == \
dbNetworkDevice.id and \
dbHardwareProfileNetwork.networkId == dbNetwork.id:
break
else:
dbHardwareProfileNetwork = HardwareProfileNetwork()
dbHardwareProfileNetwork.hardwareprofile = dbHardwareProfile
if dbNetwork.id is not None:
dbHardwareProfileNetwork.networkId = dbNetwork.id
else:
dbHardwareProfileNetwork.network = dbNetwork
dbHardwareProfileNetwork.hardwareProfileId = \
dbHardwareProfile.id
if dbNetworkDevice.id is not None:
dbHardwareProfileNetwork.networkDeviceId = \
dbNetworkDevice.id
else:
dbHardwareProfileNetwork.networkdevice = dbNetworkDevice
dbHardwareProfile.hardwareprofilenetworks.append(
dbHardwareProfileNetwork)
networks.append(dbHardwareProfileNetwork)
# Now remove all old networks
for dbNetwork in dbHardwareProfile.hardwareprofilenetworks:
for network in networks:
if network.networkDeviceId == dbNetwork.networkDeviceId \
and network.networkId == dbNetwork.networkId:
# Its a keeper
break
else:
# No match...delete time
session.delete(dbNetwork)
# Add provisioning Nics
if hardwareProfile.getProvisioningNics():
# Only one provisioning nic is possible
nic = hardwareProfile.getProvisioningNics()[0]
for prov_nic in prov_nics:
if nic.getIp() == prov_nic.ip:
dbNic = prov_nic
break
else:
raise NicNotFound(
'Provisioning NIC with IP [%s] not found' % nic.getIp())
if dbNic not in dbHardwareProfile.nics:
dbHardwareProfile.nics.append(dbNic)
return dbHardwareProfile
def __configure(self):
""" Configure all repositories from the config file. """
self._kitArchiveDir = self._cm.getKitDir()
configFile = self._cm.getRepoConfigFile()
if not os.path.exists(configFile):
self._logger.debug('Repo configuration file [%s] not found' %
(configFile))
else:
self._logger.debug('Reading repo configuration file [%s]' %
(configFile))
configParser = configparser.ConfigParser()
configParser.read(configFile)
try:
osKeyList = configParser.sections()
if osKeyList:
self._logger.debug('Found OS sections [%s]' %
(' '.join(osKeyList)))
for osKey in osKeyList:
self._logger.debug('Parsing OS section [%s]' % (osKey))
osData = osKey.split('__')
osName = osData[0]
osVersion = osData[1]
osArch = osData[2]
osInfo = OsInfo(osName, osVersion, osArch)
# Ignore all repos that weren't enabled
bEnabled = True
if configParser.has_option(osKey, 'enabled'):
value = configParser.get(osKey, 'enabled')
bEnabled = value.lower() == 'true'
if not bEnabled:
self._logger.debug('Repo for [%s] is disabled' % (osInfo))
continue
localPath = None
if configParser.has_option(osKey, 'localPath'):
localPath = configParser.get(osKey, 'localPath', True)
if not localPath:
localPath = self._repoRoot
remoteUrl = None
if configParser.has_option(osKey, 'remoteUrl'):
remoteUrl = configParser.get(osKey, 'remoteUrl', True)
repo = self.__configureRepo(osInfo, localPath, remoteUrl)
self._repoMap[osKey] = repo
if not os.path.exists(self._kitArchiveDir):
self._logger.debug('Creating kit archive directory [%s]' %
(self._kitArchiveDir))
os.makedirs(self._kitArchiveDir)
osInfo = osUtility.getNativeOsInfo()
repo = self.__configureRepo(osInfo, self._repoRoot)
osKey = '%s__%s__%s' % (osInfo.getName(), osInfo.getVersion(),
osInfo.getArch())
# Configure repo for native os if there are no repos configured.
if osKey not in self._repoMap:
self._repoMap[osKey] = repo
except ConfigurationError:
raise
except Exception as ex:
raise ConfigurationError(exception=ex)