def test_file():
fs = settings.FileSetting(must_exist=False)
#
# Assert that a non-string value raises a validation error
#
with pytest.raises(settings.SettingValidationError):
fs.validate(1)
#
# Assert that a string value does not raise a validation error
#
fs.validate('abc')
#
# Assert that if a file doesn't exist, a validation error is raised
#
fs.must_exist = True
with pytest.raises(settings.SettingValidationError):
fs.validate('abc')
#
# Assert that if a file does exist, nothing happens
#
fs.must_exist = True
fs.validate(__file__)
#
# Assert value is in values list
#
fs.must_exist = False
fs.values = ['abc', 'def']
with pytest.raises(settings.SettingValidationError):
fs.validate('ghi')
fs.validate('def')
def test_dump():
s = {
'bool': settings.BooleanSetting(),
'list_bool': settings.BooleanSetting(list=True),
'integer': settings.IntegerSetting(),
'list_integer': settings.IntegerSetting(list=True),
'string': settings.StringSetting(),
'list_string': settings.StringSetting(list=True),
'file': settings.FileSetting(must_exist=False)
}
p = ConfigurationValidator(s)
#
# Test data loading
#
data_to_load = {
'bool': 'True',
'list_bool': 'True, false',
'integer': '3',
'list_integer': '4, 5',
'string': 'abc',
'list_string': 'wz, yz',
'file': 'file.txt'
}
p.load(data_to_load)
#
# Perform full validation
#
p.validate(full=True)
#
# Assert that dumped data is properly transformed
#
expected_data = {
'bool': True,
'list_bool': [True, False],
'integer': 3,
'list_integer': [4, 5],
'string': 'abc',
'list_string': ['wz', 'yz'],
'file': 'file.txt'
}
assert p.dump() == expected_data
**GROUP_INSTANCES),
'ami':
settings.StringSetting(
display_name='AMI',
description='AMI ID to use for launching node instances',
required=True,
**GROUP_INSTANCES),
'block_device_map':
settings.StringSetting(
display_name='Block device map',
description='Used to define block devices (virtual hard drives)',
**GROUP_INSTANCES),
'cloud_init_script_template':
settings.FileSetting(display_name='cloud-init script template',
description='Path to cloud-init script template',
mutually_exclusive=['user_data_script_template'],
base_path='/opt/tortuga/config/',
overrides=['user_data_script_template'],
**GROUP_INSTANCES),
'user_data_script_template':
settings.FileSetting(display_name='User data script template',
description='Path to user data template script',
mutually_exclusive=['cloud_init_script_template'],
base_path='/opt/tortuga/config/',
overrides=['cloud_init_script_template'],
**GROUP_INSTANCES),
'vcpus':
settings.IntegerSetting(
display_name='Number of VCPUs',
description='Override automatically detected virtual CPU count',
**GROUP_INSTANCES),
'monitoring_enabled':
class Oracleadapter(ResourceAdapter):
"""
Drive Oracle Cloud Infrastructure.
"""
__adaptername__ = 'oraclecloud'
settings = {
'availability_domain': settings.StringSetting(required=True),
'compartment_id': settings.StringSetting(required=True),
'shape': settings.StringSetting(default='VM.Standard1.1'),
'vcpus': settings.IntegerSetting(),
'subnet_id': settings.StringSetting(required=True),
'image_id': settings.StringSetting(required=True),
'user_data_script_template': settings.FileSetting(
base_path='/tortuga/config/',
default='oci_bootstrap.tmpl'
),
'override_dns_domain': settings.BooleanSetting(default='False'),
'dns_options': settings.StringSetting(),
'dns_search': settings.StringSetting(),
'dns_nameservers': settings.StringSetting(
list=True,
list_separator=' '
)
}
def __init__(self, addHostSession=None):
"""
Upon instantiation, read and validate config file.
:return: Oci instance
"""
super(Oracleadapter, self).__init__(addHostSession=addHostSession)
config = {
'region': None,
'log_requests': False,
'tenancy': None,
'user': None,
'pass_phrase': None,
'fingerprint': None,
'additional_user_agent': '',
'key_file': os.path.join(os.path.expanduser('~'), '.ssh/id_rsa')
}
override_config = self.getResourceAdapterConfig()
if override_config and isinstance(override_config, dict):
config.update(override_config)
self._timeouts = {
'launch': override_config['launch_timeout'] if
'launch_timeout' in override_config else 300
}
oci.config.validate_config(config)
self.__vcpus = None
self.__installer_ip = None
self.__client = oci.core.compute_client.ComputeClient(config)
self.__net_client = \
oci.core.virtual_network_client.VirtualNetworkClient(config)
self.__identity_client = \
oci.identity.identity_client.IdentityClient(config)
def __validate_keys(self, config):
"""
Check all the required keys exist.
:param config: Dictionary
:return: None
"""
provided_keys = set(config.keys())
required_keys = {
'availability_domain',
'compartment_id',
'shape',
'subnet_id',
'image_id'
}
missing_keys = required_keys.difference(provided_keys)
if missing_keys:
error_message = \
'Required configuration setting(s) [%s] are missing' % (
' '.join(missing_keys)
)
self.getLogger().error(error_message)
@staticmethod
def __cloud_instance_metadata() -> dict:
"""
Get the cloud metadata.
:returns: Dictionary metadata
"""
response = urlopen('http://169.254.169.254/opc/v1/instance/')
return json.load(response)
@staticmethod
def __cloud_vnic_metadata() -> dict:
"""
Get the VNIC cloud metadata.
:returns: Dictionary metadata
"""
response = urlopen('http://169.254.169.254/opc/v1/vnics/')
return json.load(response)
@property
def __cloud_launch_metadata(self) -> dict:
"""
Get metadata needed to create metadata.
:returns: Dictionary metadata
"""
compute = self.__cloud_instance_metadata()
vnic = self.__cloud_vnic_metadata()
full_vnic = self.__net_client.get_vnic(
vnic['vnicId']
).data
return {
'availability_domain': compute['availabilityDomain'],
'compartment_id': compute['compartmentId'],
'subnet_id': full_vnic.subnet_id,
'image_id': compute['image'],
'region': compute['region'],
'tenancy_id': '',
'user_id': '',
'shape': compute['shape']
}
def start(self, addNodesRequest, dbSession, dbHardwareProfile,
dbSoftwareProfile=None):
"""
Create a cloud and bind with Tortuga.
:return: List Instance objects
"""
self.getLogger().debug(
'start(): addNodesRequest=[%s], dbSession=[%s],'
' dbHardwareProfile=[%s], dbSoftwareProfile=[%s]' % (
addNodesRequest,
dbSession,
dbHardwareProfile,
dbSoftwareProfile
)
)
with StopWatch() as stop_watch:
nodes = self.__add_nodes(
addNodesRequest,
dbSession,
dbHardwareProfile,
dbSoftwareProfile
)
if len(nodes) < addNodesRequest['count']:
self.getLogger().warning(
'%s node(s) requested, only %s launched'
' successfully' % (
addNodesRequest['count'],
len(nodes)
)
)
self.getLogger().debug(
'start() session [%s] completed in'
' %0.2f seconds' % (
self.addHostSession,
stop_watch.result.seconds +
stop_watch.result.microseconds / 1000000.0
)
)
self.addHostApi.clear_session_nodes(nodes)
return nodes
def __add_nodes(self, add_nodes_request, db_session, db_hardware_profile,
db_software_profile):
"""
Add nodes to the infrastructure.
:return: List Nodes objects
"""
# TODO: this validation needs to be moved
# self.__validate_keys(session.config)
node_spec = {
'db_hardware_profile': db_hardware_profile,
'db_software_profile': db_software_profile,
'db_session': db_session,
'configDict': self.getResourceAdapterConfig(),
}
return [result for result in self.__oci_add_nodes(
count=int(add_nodes_request['count']),
node_spec=node_spec)]
def __oci_add_nodes(self, count=1, node_spec=None):
"""
Wrapper around __oci_add_node() method. Launches Greenlets to
perform add nodes operation in parallel using gevent.
:param count: number of nodes to add
:param node_spec: dict containing instance launch specification
:return: list of Nodes
"""
greenlets = []
for _ in range(count):
greenlets.append(gevent.spawn(self.__oci_add_node, node_spec))
for result in gevent.iwait(greenlets):
if result.value:
yield result.value
def __oci_add_node(self, node_spec):
"""
Add one node and backing instance to Tortuga.
:param node_spec: instance launch specification
:return: Nodes object (or None, on failure)
"""
with gevent.Timeout(self._timeouts['launch'], TimeoutError):
node_dict = self.__oci_pre_launch_instance(node_spec=node_spec)
try:
instance = self._launch_instance(node_dict=node_dict,
node_spec=node_spec)
except Exception as exc:
if 'node' in node_dict:
self.__client.terminate_instance(
node_dict['instance_ocid'])
self._wait_for_instance_state(
node_dict['instance_ocid'], 'TERMINATED')
node_spec['db_session'].delete(node_dict['node'])
node_spec['db_session'].commit()
self.getLogger().error(
'Error launching instance: [{}]'.format(exc)
)
return
return self._instance_post_launch(
instance, node_dict=node_dict, node_spec=node_spec)
def __oci_pre_launch_instance(self, node_spec=None):
"""
Creates Nodes object if Tortuga-generated host names are enabled,
otherwise returns empty node dict.
:param node_spec: dict containing instance launch specification
:return: node dict
"""
if node_spec['db_hardware_profile'].nameFormat == '*':
return {}
result = {}
# Generate node name
hostname, _ = self.addHostApi.generate_node_name(
node_spec['db_session'],
node_spec['db_hardware_profile'].nameFormat,
dns_zone=self.private_dns_zone).split('.', 1)
_, domain = self.installer_public_hostname.split('.', 1)
name = '%s.%s' % (hostname, domain)
# Create Nodes object
node = self.__initialize_node(
name,
node_spec['db_hardware_profile'],
node_spec['db_software_profile']
)
node.state = state.NODE_STATE_LAUNCHING
result['node'] = node
# Add to database and commit database session
node_spec['db_session'].add(node)
node_spec['db_session'].commit()
return result
def __initialize_node(self, name, db_hardware_profile,
db_software_profile):
node = Node(name=name)
node.softwareprofile = db_software_profile
node.hardwareprofile = db_hardware_profile
node.isIdle = False
node.addHostSession = self.addHostSession
return node
def _launch_instance(self, node_dict=None, node_spec=None):
"""
Launch instance and wait for it to reach RUNNING state.
:param node_dict: Dictionary
:param node_spec: Object
:return: Instance object
"""
session = OciSession(node_spec['configDict'])
session.config['metadata']['user_data'] = \
self.__get_user_data(session.config)
# TODO: this is a temporary workaround until the OciSession
# functionality is validated for this workflow
launch_config = session.launch_config
# TODO: make this work better. Need to
# find a way of injecting this into the
# `get_node_vcpus` method.
self.__vcpus = session.config['vcpus'] if \
session.config['vcpus'] else \
session.cores_from_shape
self.getLogger().debug(
'setting vcpus to %d' % (
self.__vcpus
)
)
if 'node' in node_dict:
node = node_dict['node']
self.getLogger().debug(
'overriding instance name [%s]' % (
node.name)
)
launch_config.display_name = node.name
launch_config.hostname_label = node.name.split('.', 1)[0]
launch_instance = self.__client.launch_instance(launch_config)
instance_ocid = launch_instance.data.id
node_dict['instance_ocid'] = instance_ocid
log_adapter = CustomAdapter(
self.getLogger(), {'instance_ocid': instance_ocid})
log_adapter.debug('launched')
# TODO: implement a timeout waiting for an instance to start; this
# will currently wait forever
# TODO: check for launch error
def logging_callback(instance, state):
log_adapter.debug('state: %s; waiting...' % state)
self._wait_for_instance_state(
instance_ocid, 'RUNNING', callback=logging_callback)
log_adapter.debug('state: RUNNING')
return self.__client.get_instance(instance_ocid).data
def get_node_vcpus(self, name):
"""
Return resolved number of VCPUs.
:param name: String node hostname
:return: Integer vcpus
"""
instance_cache = self.instanceCacheGet(name)
if 'vcpus' in list(instance_cache.keys()):
return int(instance_cache['vcpus'])
return self.__vcpus
def _instance_post_launch(self, instance, node_dict=None, node_spec=None):
"""
Called after instance has launched successfully.
:param instance: Oracle instance
:param node_dict: instance/node mapping dict
:param node_spec: instance launch specification
:return: Nodes object
"""
self.getLogger().debug(
'Instance post-launch action for instance [%s]' % (
instance.id)
)
if 'node' not in node_dict:
domain = self.installer_public_hostname.split('.')[1:]
fqdn = '.'.join([instance.display_name] + domain)
node = self.__initialize_node(
fqdn,
node_spec['db_hardware_profile'],
node_spec['db_software_profile']
)
node_spec['db_session'].add(node)
node_dict['node'] = node
else:
node = node_dict['node']
node.state = state.NODE_STATE_PROVISIONED
# Get ip address from instance
nics = []
for ip in self.__get_instance_private_ips(
instance.id, instance.compartment_id):
nics.append(
Nic(ip=ip, boot=True)
)
node.nics = nics
node_spec['db_session'].commit()
self.instanceCacheSet(
node.name,
{
'id': instance.id,
'compartment_id': instance.id,
'shape': node_spec['configDict']['shape'],
'vcpus': str(node_spec['configDict']['shape'].split('.')[-1])
}
)
ip = [nic for nic in node.nics if nic.boot][0].ip
self._pre_add_host(
node.name,
node.hardwareprofile.name,
node.softwareprofile.name,
ip)
self.getLogger().debug(
'_instance_post_launch(): node=[%s]' % (
node)
)
self.fire_provisioned_event(node)
return node
def __get_instance_public_ips(self, instance_id, compartment_id):
"""
Get public IP from the attached VNICs.
:param instance_id: String instance id
:param compartment_id: String compartment id
:return: Generator String IPs
"""
for vnic in self.__get_vnics_for_instance(instance_id, compartment_id):
attached_vnic = self.__net_client.get_vnic(vnic.vnic_id)
if attached_vnic:
yield attached_vnic.data.public_ip
def __get_instance_private_ips(self, instance_id, compartment_id):
"""
Get private IP from the attached VNICs.
:param instance_id: String instance id
:param compartment_id: String compartment id
:return: Generator String IPs
"""
for vnic in self.__get_vnics_for_instance(instance_id, compartment_id):
attached_vnic = self.__net_client.get_vnic(vnic.vnic_id)
if attached_vnic:
yield attached_vnic.data.private_ip
def __get_vnics_for_instance(self, instance_id, compartment_id):
"""
Get all VNICs attached to instance.
:param instance_id: String instance id
:param compartment_id: String compartment id
:return: Generator VNIC objects
"""
for vnic in self.__get_vnics(compartment_id):
if vnic.instance_id == instance_id \
and vnic.lifecycle_state == 'ATTACHED':
yield vnic
def __get_vnics(self, compartment_id):
"""
Get VNICs in compartment.
:param compartment_id: String id
:return: List VNIC objects
"""
vnics = self.__client.list_vnic_attachments(compartment_id)
return vnics.data
def __get_common_user_data_settings(self, config, node=None):
"""
Format resource adapters for the bootstrap
template.
:param config: Dictionary
:param node: Node instance
:return: Dictionary
"""
installer_ip = self.__get_installer_ip(
hardwareprofile=node.hardwareprofile if node else None)
settings_dict = {
'installerHostName': self.installer_public_hostname,
'installerIp': '\'{0}\''.format(installer_ip)
if installer_ip else 'None',
'adminport': self._cm.getAdminPort(),
'cfmuser': self._cm.getCfmUser(),
'cfmpassword': self._cm.getCfmPassword(),
'override_dns_domain': str(config['override_dns_domain']),
'dns_options': '\'{0}\''.format(config['dns_options'])
if config['dns_options'] else None,
'dns_search': '\'{0}\''.format(config['dns_search'])
if config['dns_search'] else None,
'dns_nameservers': self.__get_encoded_list(
config['dns_nameservers']),
}
return settings_dict
def __get_common_user_data_content(self, settings_dict):
"""
Create header for bootstrap file.
:param settings_dict: Dictionary
:return: String
"""
result = """\
installerHostName = '%(installerHostName)s'
installerIpAddress = %(installerIp)s
port = %(adminport)d
cfmUser = '******'
cfmPassword = '******'
# DNS resolution settings
override_dns_domain = %(override_dns_domain)s
dns_options = %(dns_options)s
dns_search = %(dns_search)s
dns_nameservers = %(dns_nameservers)s
""" % settings_dict
return result
def __get_user_data(self, config, node=None):
"""
Compile the cloud-init script from
bootstrap template and encode into
base64.
:param config: Dictionary
:param node: Node instance
:return: String
"""
self.getLogger().info(
'Using cloud-init script template [%s]' % (
config['user_data_script_template']))
settings_dict = self.__get_common_user_data_settings(config, node)
with open(config['user_data_script_template']) as fp:
result = ''
for line in fp.readlines():
if line.startswith('### SETTINGS'):
result += self.__get_common_user_data_content(
settings_dict)
else:
result += line
combined_message = MIMEMultipart()
if node and not config['use_instance_hostname']:
# Use cloud-init to set fully-qualified domain name of instance
cloud_init = """#cloud-config
fqdn: %s
""" % node.name
sub_message = MIMEText(
cloud_init, 'text/cloud-config', sys.getdefaultencoding())
filename = 'user-data.txt'
sub_message.add_header(
'Content-Disposition',
'attachment; filename="%s"' % filename)
combined_message.attach(sub_message)
sub_message = MIMEText(
result, 'text/x-shellscript', sys.getdefaultencoding())
filename = 'bootstrap.py'
sub_message.add_header(
'Content-Disposition',
'attachment; filename="%s"' % filename)
combined_message.attach(sub_message)
return b64encode(str(combined_message).encode()).deocde()
# Fallback to default behaviour
return b64encode(result.encode()).decode()
def deleteNode(self, dbNodes):
"""
Delete a node from the infrastructure.
:param dbNodes: List Nodes object
:return: None
"""
self._async_delete_nodes(dbNodes)
self.getLogger().info(
'%d node(s) deleted' % (
len(dbNodes))
)
def _wait_for_instance_state(self, instance_ocid, state, callback=None,
timeout=None):
"""
Wait for instance to reach state
:param instance_ocid: Instance OCID
:param state: Expected state of instance
:param timeout: (optional) operation timeout
:return: None
"""
# TODO: implement timeout
for nRetries in itertools.count(0):
instance = self.__client.get_instance(instance_ocid)
if instance.data.lifecycle_state == state:
break
if callback:
# Only call the callback if the requested state hasn't yet been
# reached
callback(instance_ocid, instance.data.lifecycle_state)
gevent.sleep(get_random_sleep_time(retries=nRetries) / 1000.0)
def _delete_node(self, node):
# TODO: add error handling; if the instance termination request
# fails, we shouldn't be removing the node from the system
try:
instance_cache = self.instanceCacheGet(node.name)
# TODO: what happens when you attempt to terminate an already
# terminated instance? Exception?
instance = self.__client.get_instance(instance_cache['id'])
log_adapter = CustomAdapter(
self.getLogger(), {'instance_ocid': instance_cache['id']})
# Issue terminate request
log_adapter.debug('Terminating...')
self.__client.terminate_instance(instance.data.id)
# Wait 3 seconds before checking state
gevent.sleep(3)
# Wait until state is 'TERMINATED'
self._wait_for_instance_state(instance_cache['id'], 'TERMINATED')
# Clean up the instance cache.
self.instanceCacheDelete(node.name)
except ResourceNotFound:
pass
# Remove Puppet certificate
bhm = osUtility.getOsObjectFactory().getOsBootHostManager()
bhm.deleteNodeCleanup(node)
def __get_installer_ip(self, hardwareprofile=None):
"""
Get IP address of the installer node.
:param hardwareprofile: Object
:return: String ip address
"""
if self.__installer_ip is None:
if hardwareprofile and hardwareprofile.nics:
self.__installer_ip = hardwareprofile.nics[0].ip
else:
self.__installer_ip = self.installer_public_ipaddress
return self.__installer_ip
@staticmethod
def __get_encoded_list(items):
"""
Return Python list encoded in a string.
:param items: List
:return: String
"""
return '[' + ', '.join(['\'%s\'' % item for item in items]) + ']' \
if items else '[]'
description='Family of image used when creating compute nodes',
mutually_exclusive=['image', 'image_url'],
overrides=['image', 'image_url'],
**GROUP_INSTANCES
),
'tags': settings.TagListSetting(
display_name='Tags',
description='A comma-separated list of tags in the form of '
'key=value',
**GROUP_INSTANCES
),
'startup_script_template': settings.FileSetting(
display_name='Start-up Script Template',
required=True,
description='Filename of "bootstrap" script used by Tortuga to '
'bootstrap compute nodes',
default='startup_script.py',
base_path='/opt/tortuga/config/',
**GROUP_INSTANCES
),
'default_ssh_user': settings.StringSetting(
display_name='Default SSH User',
required=True,
description='Username of default user on created VMs. "centos" '
'is an appropriate value for CentOS-based VMs.',
**GROUP_INSTANCES
),
'vcpus': settings.IntegerSetting(
display_name='VCPUs',
description='Number of virtual CPUs for specified virtual '
'machine type',
mutually_exclusive=['image'],
overrides=['image'],
**GROUP_INSTANCES),
'image':
settings.StringSetting(display_name='Image',
description='Name of VM image',
mutually_exclusive=['image_urn'],
overrides=['image_urn'],
**GROUP_INSTANCES),
'cloud_init_script_template':
settings.FileSetting(
display_name='Cloud Init Script Template',
description='Use this setting to specify the filename/path of'
'the cloud-init script template. If the path is not'
'fully-qualified (does not start with a leading'
'forward slash), it is assumed the script path is '
'$TORTUGA_ROOT/config',
base_path='/opt/tortuga/config/',
mutually_exclusive=['user_data_script_template'],
overrides=['user_data_script_template'],
**GROUP_INSTANCES),
'user_data_script_template':
settings.FileSetting(
display_name='User Data Script Template',
description='File name of bootstrap script template to be used '
'on compute nodes. If the path is not '
'fully-qualified (ie. does not start with a leading '
'forward slash), it is assumed the script path is '
'$TORTUGA_ROOT/config',
base_path='/opt/tortuga/config/',
mutually_exclusive=['cloud_init_script_template'],
class Default(ResourceAdapter):
__adaptername__ = 'default'
settings: Dict[str, ra_settings.BaseSetting] = {
'boot_host_hook_script': ra_settings.FileSetting()
}
def __init__(self, addHostSession: Optional[str] = None) -> None:
super().__init__(addHostSession=addHostSession)
self._bhm = \
osUtility.getOsObjectFactory().getOsBootHostManager(self._cm)
@property
def hookScript(self):
'''
Load the hook script setting from the resource adapter
configuration file and ensure it exists. If the hook script is
not defined or is defined and does note exist, a warning message
is logged and the method returns None.
'''
RA_SECTION = 'resource-adapter'
OPTION = 'host_hook_script'
cfgFile = configparser.ConfigParser()
cfgFile.read(self.cfgFileName)
if not cfgFile.has_section(RA_SECTION) or \
not cfgFile.has_option(RA_SECTION, OPTION):
self._logger.warning('Hook script is not defined')
return None
hookScript = cfgFile.get(RA_SECTION, OPTION)
if not hookScript[0] in ['/', '$']:
tmpHookScript = os.path.join(self._cm.getKitConfigBase(),
hookScript)
else:
tmpHookScript = hookScript.replace('$TORTUGA_ROOT',
self._cm.getRoot())
if not os.path.join(tmpHookScript):
self._logger.warning('Hook script [%s] does not exist' %
(tmpHookScript))
return None
return tmpHookScript
def hookAction(self, action, nodes, args=None):
'''
WARNING: this method may be subject to scalability concerns if
batch node operations are implemented.
'''
hookScript = self.hookScript
if not hookScript:
return
nodeArg = nodes if not isinstance(nodes, list) else ','.join(nodes)
cmd = '%s %s' % (hookScript, action)
if args:
cmd += ' %s' % (args)
cmd += ' %s' % (nodeArg)
tortugaSubprocess.executeCommandAndIgnoreFailure(cmd)
def deleteNode(self, nodes: List[Node]) -> None:
"""Remove boot configuration for deleted nodes
"""
for node in nodes:
self.__delete_boot_configuration(node)
self.hookAction('delete', [node.name for node in nodes])
def __delete_boot_configuration(self, node: Node) -> None:
"""Remove PXE boot files and DHCP configuration
"""
self._bhm.rmPXEFile(node)
self._bhm.removeDhcpLease(node)
def rebootNode(self,
nodes: List[Node],
bSoftReset: Optional[bool] = False):
self._logger.debug('rebootNode()')
# Call the reboot script hook
self.hookAction('reset', [node.name for node in nodes],
'soft' if bSoftReset else 'hard')
def __get_node_details(self, addNodesRequest, dbHardwareProfile,
dbSoftwareProfile): \
# pylint: disable=no-self-use,unused-argument
nodeDetails = addNodesRequest['nodeDetails'] \
if 'nodeDetails' in addNodesRequest else []
# Check if any interface has predefined MAC address
macSpecified = nodeDetails and \
'nics' in nodeDetails[0] and \
[nic_ for nic_ in nodeDetails[0]['nics'] if 'mac' in nic_]
# Check if any intercace has predefined IP address
ipAddrSpecified = nodeDetails and \
'nics' in nodeDetails[0] and \
[ip_ for ip_ in nodeDetails[0]['nics'] if 'ip' in ip_]
hostNameSpecified = nodeDetails and 'name' in nodeDetails[0]
if macSpecified or ipAddrSpecified or hostNameSpecified:
return nodeDetails
return None
def start(self,
addNodesRequest,
dbSession,
dbHardwareProfile,
dbSoftwareProfile=None) -> List[Node]:
"""
Raises:
CommandFailed
"""
# 'nodeDetails' is a list of details (contained in a 'dict') for
# one or more nodes. It can contain host name(s) and nic details
# like MAC and/or IP address. It is an entirely optional data
# structure and may be empty and/or undefined.
nodeDetails = self.__get_node_details(addNodesRequest,
dbHardwareProfile,
dbSoftwareProfile)
if not nodeDetails:
raise CommandFailed('Invalid operation (DHCP discovery)')
try:
dns_zone = GlobalParametersDbHandler().getParameter(
dbSession, 'DNSZone').value
except ParameterNotFound:
dns_zone = ''
nodes = self.__add_predefined_nodes(addNodesRequest,
dbSession,
dbHardwareProfile,
dbSoftwareProfile,
dns_zone=dns_zone)
# This is a necessary evil for the time being, until there's
# a proper context manager implemented.
self.addHostApi.clear_session_nodes(nodes)
return nodes
def validate_start_arguments(self, addNodesRequest: dict,
dbHardwareProfile: HardwareProfile,
dbSoftwareProfile: SoftwareProfile): \
# pylint: disable=unused-argument,no-self-use
'''
:raises CommandFailed:
:raises NodeAlreadyExists:
'''
if dbSoftwareProfile is None:
raise CommandFailed(
"Software profile must be provided when adding nodes"
" to this hardware profile")
if dbHardwareProfile.location != 'local':
# Only ensure that required installer components are enabled when
# hardware profile is marked as 'local'.
return
# All resource adapters are responsible for doing their own
# check for the configuration. This may change in the
# future!
dbInstallerNode = dbHardwareProfile.nics[0].node \
if dbHardwareProfile.nics else \
NodesDbHandler().getNode(self.session, self._cm.getInstaller())
components = [
c for c in dbInstallerNode.softwareprofile.components
if c.name == 'dhcpd'
]
if not components:
raise CommandFailed('dhcpd component must be enabled on the'
' installer in order to provision local nodes')
name_expected = dbHardwareProfile.nameFormat == '*'
nodeDetails = addNodesRequest['nodeDetails'] \
if 'nodeDetails' in addNodesRequest and \
addNodesRequest['nodeDetails'] else None
# extract host name from addNodesRequest
name = nodeDetails[0]['name'] \
if nodeDetails and 'name' in nodeDetails[0] else None
mac_addr = None
if nodeDetails and 'nics' in nodeDetails[0]:
for nic in nodeDetails[0]['nics']:
if 'mac' in nic:
mac_addr = nic['mac']
break
# check if name is expected in nodeDetails
if not nodeDetails and name_expected and not name:
raise CommandFailed(
'Name and MAC address must be specified for nodes'
' in hardware profile [%s]' % dbHardwareProfile.name)
if not nodeDetails and not mac_addr:
raise CommandFailed('MAC address must be specified for nodes in'
' hardware profile [%s]' %
dbHardwareProfile.name)
# if host name specified, ensure host does not already exist
if name:
try:
NodesDbHandler().getNode(self.session, name)
raise NodeAlreadyExists('Node [%s] already exists' % name)
except NodeNotFound:
# node does not already exist
pass
def __add_predefined_nodes(self,
addNodesRequest: dict,
dbSession,
dbHardwareProfile,
dbSoftwareProfile,
dns_zone: str = None) -> List[Node]:
nodeDetails = addNodesRequest['nodeDetails'] \
if 'nodeDetails' in addNodesRequest else []
bGenerateIp = dbHardwareProfile.location != 'remote'
newNodes = []
for nodeDict in nodeDetails:
addNodeRequest = {}
addNodeRequest['addHostSession'] = self.addHostSession
if 'rack' in addNodesRequest:
# rack can be undefined, in which case it is not copied
# into the node request
addNodeRequest['rack'] = addNodesRequest['rack']
if 'nics' in nodeDict:
addNodeRequest['nics'] = nodeDict['nics']
if 'name' in nodeDict:
addNodeRequest['name'] = nodeDict['name']
node = self.nodeApi.createNewNode(dbSession,
addNodeRequest,
dbHardwareProfile,
dbSoftwareProfile,
bGenerateIp=bGenerateIp,
dns_zone=dns_zone)
dbSession.add(node)
# Create DHCP/PXE configuration
self.writeLocalBootConfiguration(node, dbHardwareProfile,
dbSoftwareProfile)
# Get the provisioning nic
nics = get_provisioning_nics(node)
self._pre_add_host(node.name, dbHardwareProfile.name,
dbSoftwareProfile.name,
nics[0].ip if nics else None)
newNodes.append(node)
return newNodes
def stop(self, hardwareProfileName, deviceName): \
# pylint: disable=unused-argument
pass
def addVolumeToNode(self, node, volume, isDirect=True):
'''Add a disk to a node'''
if not isDirect:
raise UnsupportedOperation(
'This node only supports direct volume attachment.')
# Map the volume to a driveNumber
openDriveNumber = self.sanApi.mapDrive(node, volume)
try:
# have the node connect the storage
self.sanApi.connectStorageVolume(node, volume, node.getName())
except Exception: # noqa pylint: disable=broad-except
self._logger.exception('Error adding volume to node')
# Need to clean up mapping
self.sanApi.unmapDrive(node, driveNumber=openDriveNumber)
raise
def removeVolumeFromNode(self, node, volume):
'''Remove a disk from a node'''
try:
try:
self.sanApi.disconnectStorageVolume(node, volume,
node.getName())
except Exception: # noqa pylint: disable=broad-except
# Failed disconnect...
self._logger.exception('Error disconnecting volume from node')
raise
finally:
# Unmap Map the volume to a driveNumber
self.sanApi.unmapDrive(node, volume=volume)
def shutdownNode(self,
nodes: List[Node],
bSoftReset: Optional[bool] = False):
"""
Shutdown specified node(s)
"""
self.hookAction('shutdown', [node.name for node in nodes],
'soft' if bSoftReset else 'hard')
def startupNode(self,
nodes: List[Node],
remainingNodeList: Optional[str] = None,
tmpBootMethod: Optional[str] = 'n'): \
# pylint: disable=unused-argument
"""
Start the given node(s)
"""
self.hookAction('start', [node.name for node in nodes])
class Default(ResourceAdapter):
__adaptername__ = 'default'
settings: Dict[str, ra_settings.BaseSetting] = {
'boot_host_hook_script': ra_settings.FileSetting()
}
def __init__(self, addHostSession: Optional[str] = None) -> None:
super().__init__(addHostSession=addHostSession)
self._bhm = \
osUtility.getOsObjectFactory().getOsBootHostManager(self._cm)
self.looping = False
@property
def hookScript(self):
'''
Load the hook script setting from the resource adapter
configuration file and ensure it exists. If the hook script is
not defined or is defined and does note exist, a warning message
is logged and the method returns None.
'''
RA_SECTION = 'resource-adapter'
OPTION = 'host_hook_script'
cfgFile = configparser.ConfigParser()
cfgFile.read(self.cfgFileName)
if not cfgFile.has_section(RA_SECTION) or \
not cfgFile.has_option(RA_SECTION, OPTION):
self.getLogger().warning('Hook script is not defined')
return None
hookScript = cfgFile.get(RA_SECTION, OPTION)
if not hookScript[0] in ['/', '$']:
tmpHookScript = os.path.join(
self._cm.getKitConfigBase(), hookScript)
else:
tmpHookScript = hookScript.replace(
'$TORTUGA_ROOT', self._cm.getRoot())
if not os.path.join(tmpHookScript):
self.getLogger().warning(
'Hook script [%s] does not exist' % (tmpHookScript))
return None
return tmpHookScript
def hookAction(self, action, nodes, args=None):
'''
WARNING: this method may be subject to scalability concerns if
batch node operations are implemented.
'''
hookScript = self.hookScript
if not hookScript:
return
nodeArg = nodes if not isinstance(nodes, list) else ','.join(nodes)
cmd = '%s %s' % (hookScript, action)
if args:
cmd += ' %s' % (args)
cmd += ' %s' % (nodeArg)
tortugaSubprocess.executeCommandAndIgnoreFailure(cmd)
def transferNode(self, nodeIdSoftwareProfileTuples,
newSoftwareProfileName): \
# pylint: disable=unused-argument
"""
Raises:
NodeNotFound
"""
for dbNode, _ in nodeIdSoftwareProfileTuples:
# Ensure PXE files are properly in place before triggering
# the reboot.
self._bhm.setNodeForNetworkBoot(self.session, dbNode)
self.rebootNode([dbNode for dbNode, _ in nodeIdSoftwareProfileTuples])
def suspendActiveNode(self, node: Node) -> bool: \
# pylint: disable=no-self-use,unused-argument
# not supported
return False
def idleActiveNode(self, nodes: List[Node]) -> str:
# Shutdown nodes
self.shutdownNode(nodes)
return 'Discovered'
def activateIdleNode(self, node: Node, softwareProfileName: str,
softwareProfileChanged: bool):
# pylint: disable=no-self-use
if softwareProfileChanged:
softwareprofile = \
SoftwareProfilesDbHandler().getSoftwareProfile(
self.session, softwareProfileName)
# Mark node for network boot if software profile changed
node.bootFrom = 0
else:
softwareprofile = None
self._bhm.writePXEFile(
self.session, node, localboot=not softwareProfileChanged,
softwareprofile=softwareprofile
)
def abort(self):
self.looping = False
def deleteNode(self, nodes: List[Node]) -> None:
self.hookAction('delete', [node.name for node in nodes])
def rebootNode(self, nodes: List[Node],
bSoftReset: Optional[bool] = False):
self.getLogger().debug('rebootNode()')
# Call the reboot script hook
self.hookAction('reset', [node.name for node in nodes],
'soft' if bSoftReset else 'hard')
def __is_duplicate_mac_in_session(self, mac, session_nodes): \
# pylint: disable=no-self-use
for node in session_nodes:
for nic in node.nics:
if nic.mac == mac:
return True
return False
def __is_duplicate_mac(self, mac, session_nodes):
if self.__is_duplicate_mac_in_session(mac, session_nodes):
return True
try:
NodesDbHandler().getNodeByMac(self.session, mac)
return True
except NodeNotFound:
return False
def __get_node_details(self, addNodesRequest, dbHardwareProfile,
dbSoftwareProfile): \
# pylint: disable=no-self-use,unused-argument
nodeDetails = addNodesRequest['nodeDetails'] \
if 'nodeDetails' in addNodesRequest else []
# Check if any interface has predefined MAC address
macSpecified = nodeDetails and \
'nics' in nodeDetails[0] and \
[nic_ for nic_ in nodeDetails[0]['nics'] if 'mac' in nic_]
# Check if any intercace has predefined IP address
ipAddrSpecified = nodeDetails and \
'nics' in nodeDetails[0] and \
[ip_ for ip_ in nodeDetails[0]['nics'] if 'ip' in ip_]
hostNameSpecified = nodeDetails and 'name' in nodeDetails[0]
if macSpecified or ipAddrSpecified or hostNameSpecified:
return nodeDetails
return None
def start(self, addNodesRequest, dbSession, dbHardwareProfile,
dbSoftwareProfile=None) -> List[Node]:
"""
Raises:
CommandFailed
"""
# 'nodeDetails' is a list of details (contained in a 'dict') for
# one or more nodes. It can contain host name(s) and nic details
# like MAC and/or IP address. It is an entirely optional data
# structure and may be empty and/or undefined.
nodeDetails = self.__get_node_details(
addNodesRequest, dbHardwareProfile, dbSoftwareProfile)
# return self.__add_predefined_nodes(
# addNodesRequest, dbSession, dbHardwareProfile,
# dbSoftwareProfile) \
# if nodeDetails else self.__dhcp_discovery(
# addNodesRequest, dbSession, dbHardwareProfile,
# dbSoftwareProfile)
if not nodeDetails:
raise CommandFailed('Invalid operation (DHCP discovery)')
try:
dns_zone = GlobalParametersDbHandler().getParameter(
dbSession, 'DNSZone').value
except ParameterNotFound:
dns_zone = ''
nodes = self.__add_predefined_nodes(
addNodesRequest, dbSession, dbHardwareProfile, dbSoftwareProfile,
dns_zone=dns_zone)
# This is a necessary evil for the time being, until there's
# a proper context manager implemented.
self.addHostApi.clear_session_nodes(nodes)
return nodes
def validate_start_arguments(self, addNodesRequest, dbHardwareProfile,
dbSoftwareProfile): \
# pylint: disable=unused-argument,no-self-use
'''
:raises CommandFailed:
:raises NodeAlreadyExists:
'''
if dbSoftwareProfile is None:
raise CommandFailed(
"Software profile must be provided when adding nodes"
" to this hardware profile")
if dbHardwareProfile.location != 'local':
# Only ensure that required installer components are enabled when
# hardware profile is marked as 'local'.
return
# All resource adapters are responsible for doing their own
# check for the configuration. This may change in the
# future!
dbInstallerNode = dbHardwareProfile.nics[0].node \
if dbHardwareProfile.nics else \
NodesDbHandler().getNode(self.session, self._cm.getInstaller())
components = [c for c in dbInstallerNode.softwareprofile.components
if c.name == 'dhcpd']
if not components:
raise CommandFailed(
'dhcpd component must be enabled on the'
' installer in order to provision local nodes')
name_expected = dbHardwareProfile.nameFormat == '*'
nodeDetails = addNodesRequest['nodeDetails'] \
if 'nodeDetails' in addNodesRequest and \
addNodesRequest['nodeDetails'] else None
# extract host name from addNodesRequest
name = nodeDetails[0]['name'] \
if nodeDetails and 'name' in nodeDetails[0] else None
mac_addr = None
if nodeDetails and 'nics' in nodeDetails[0]:
for nic in nodeDetails[0]['nics']:
if 'mac' in nic:
mac_addr = nic['mac']
break
# check if name is expected in nodeDetails
if not nodeDetails and name_expected and not name:
raise CommandFailed(
'Name and MAC address must be specified for nodes'
' in hardware profile [%s]' % dbHardwareProfile.name
)
if not nodeDetails and not mac_addr:
raise CommandFailed(
'MAC address must be specified for nodes in'
' hardware profile [%s]' % dbHardwareProfile.name
)
# if host name specified, ensure host does not already exist
if name:
try:
NodesDbHandler().getNode(self.session, name)
raise NodeAlreadyExists('Node [%s] already exists' % name)
except NodeNotFound:
# node does not already exist
pass
def __add_predefined_nodes(self, addNodesRequest: dict, dbSession,
dbHardwareProfile, dbSoftwareProfile,
dns_zone: str = None) -> List[Node]:
nodeDetails = addNodesRequest['nodeDetails'] \
if 'nodeDetails' in addNodesRequest else []
bGenerateIp = dbHardwareProfile.location != 'remote'
newNodes = []
for nodeDict in nodeDetails:
addNodeRequest = {}
addNodeRequest['addHostSession'] = self.addHostSession
if 'rack' in addNodesRequest:
# rack can be undefined, in which case it is not copied
# into the node request
addNodeRequest['rack'] = addNodesRequest['rack']
if 'nics' in nodeDict:
addNodeRequest['nics'] = nodeDict['nics']
if 'name' in nodeDict:
addNodeRequest['name'] = nodeDict['name']
node = self.nodeApi.createNewNode(
dbSession, addNodeRequest, dbHardwareProfile,
dbSoftwareProfile, bGenerateIp=bGenerateIp, dns_zone=dns_zone)
dbSession.add(node)
# Create DHCP/PXE configuration
self.writeLocalBootConfiguration(
node, dbHardwareProfile, dbSoftwareProfile)
# Get the provisioning nic
nics = get_provisioning_nics(node)
self._pre_add_host(
node.name,
dbHardwareProfile.name,
dbSoftwareProfile.name,
nics[0].ip if nics else None)
newNodes.append(node)
return newNodes
def __dhcp_discovery(self, addNodesRequest, dbSession, dbHardwareProfile,
dbSoftwareProfile):
# Listen for DHCP requests
if not dbHardwareProfile.nics:
raise CommandFailed(
'Hardware profile [%s] does not have a provisioning'
' NIC defined' % (dbHardwareProfile.name))
newNodes = []
deviceName = addNodesRequest['deviceName'] \
if 'deviceName' in addNodesRequest else None
nodeCount = addNodesRequest['count'] \
if 'count' in addNodesRequest else 0
bGenerateIp = dbHardwareProfile.location != 'remote'
# Obtain platform-specific packet capture subprocess object
addHostManager = osUtility.getOsObjectFactory().getOsAddHostManager()
deviceName = dbHardwareProfile.nics[0].networkdevice.name
p1 = addHostManager.dhcpCaptureSubprocess(deviceName)
if nodeCount:
self.getLogger().debug(
'Adding [%s] new %s' % (
nodeCount, 'nodes' if nodeCount > 1 else 'node'))
self.looping = True
index = 0
# Node count was not specified, so discover DHCP nodes
# until manually aborted by user.
msg = 'Waiting for new node...' if not nodeCount else \
'Waiting for new node #1 of %d...' % (nodeCount)
try:
while self.looping:
# May not need this...
dataReady = select.select([p1.stdout], [], [], 5)
if not dataReady[0]:
continue
line = p1.stdout.readline()
if not line:
self.getLogger().debug(
'DHCP packet capture process ended... exiting')
break
self.getLogger().debug(
'Read line "%s" len=%s' % (line, len(line)))
mac = addHostManager.getMacAddressFromCaptureEntry(line)
if not mac:
continue
self.getLogger().debug('Discovered MAC address [%s]' % (mac))
if self.__is_duplicate_mac(mac, newNodes):
# Ignore DHCP request from known MAC
self.getLogger().debug(
'MAC address [%s] is already known' % (mac))
continue
addNodeRequest = {}
if 'rack' in addNodesRequest:
addNodeRequest['rack'] = addNodesRequest['rack']
# Get nics based on hardware profile networks
addNodeRequest['nics'] = initialize_nics(
dbHardwareProfile.nics[0],
dbHardwareProfile.hardwareprofilenetworks, mac)
# We may be trying to create the same node for the
# second time so we'll ignore errors
try:
node = self.nodeApi.createNewNode(
None,
addNodeRequest,
dbHardwareProfile,
dbSoftwareProfile,
bGenerateIp=bGenerateIp)
except NodeAlreadyExists as ex:
existingNodeName = ex.args[0]
self.getLogger().debug(
'Node [%s] already exists' % (existingNodeName))
continue
except MacAddressAlreadyExists:
self.getLogger().debug(
'MAC address [%s] already exists' % (mac))
continue
except IpAlreadyExists as ex:
self.getLogger().debug(
'IP address already in use by node'
' [%s]: %s' % (existingNodeName, ex))
continue
# Add the newly created node to the session
dbSession.add(node)
# Create DHCP/PXE configuration
self.writeLocalBootConfiguration(
node, dbHardwareProfile, dbSoftwareProfile)
index += 1
# Use first provisioning nic
nic = get_provisioning_nic(node)
try:
msg = 'Added node [%s] IP [%s]' % (
node.name, nic.ip)
if nic.mac:
msg += ' MAC [%s]' % (nic.mac)
self.getLogger().info(msg)
except Exception as ex: # noqa pylint: disable=broad-except
self.getLogger().exception('Error setting status message')
self._pre_add_host(
node.name,
dbHardwareProfile.name,
dbSoftwareProfile.name,
nic.ip)
newNodes.append(node)
if nodeCount > 0:
nodeCount -= 1
if not nodeCount:
self.looping = False
except Exception as msg: # noqa pylint: disable=broad-except
self.getLogger().exception('DHCP discovery failed')
try:
os.kill(p1.pid, signal.SIGKILL)
os.waitpid(p1.pid, 0)
except Exception: # noqa pylint: disable=broad-except
self.getLogger().exception(
'Error killing network capture process')
# This is a necessary evil for the time being, until there's
# a proper context manager implemented.
self.addHostApi.clear_session_nodes(newNodes)
return newNodes
def stop(self, hardwareProfileName, deviceName): \
# pylint: disable=unused-argument
pass
def addVolumeToNode(self, node, volume, isDirect=True):
'''Add a disk to a node'''
if not isDirect:
raise UnsupportedOperation(
'This node only supports direct volume attachment.')
# Map the volume to a driveNumber
openDriveNumber = self.sanApi.mapDrive(node, volume)
try:
# have the node connect the storage
self.sanApi.connectStorageVolume(node, volume, node.getName())
except Exception: # noqa pylint: disable=broad-except
self.getLogger().exception('Error adding volume to node')
# Need to clean up mapping
self.sanApi.unmapDrive(node, driveNumber=openDriveNumber)
raise
def removeVolumeFromNode(self, node, volume):
'''Remove a disk from a node'''
try:
try:
self.sanApi.disconnectStorageVolume(
node, volume, node.getName())
except Exception: # noqa pylint: disable=broad-except
# Failed disconnect...
self.getLogger().exception(
'Error disconnecting volume from node')
raise
finally:
# Unmap Map the volume to a driveNumber
self.sanApi.unmapDrive(node, volume=volume)
def shutdownNode(self, nodes: List[Node],
bSoftReset: Optional[bool] = False):
"""
Shutdown specified node(s)
"""
self.hookAction(
'shutdown', [node.name for node in nodes],
'soft' if bSoftReset else 'hard')
def startupNode(self, nodes: List[Node],
remainingNodeList: Optional[str] = None,
tmpBootMethod: Optional[str] = 'n'): \
# pylint: disable=unused-argument
"""
Start the given node(s)
"""
self.hookAction('start', [node.name for node in nodes])