def _finish_set_ipmi_credentials(ironic, node, cached_node, node_info,
new_username, new_password):
patch = [{'op': 'add', 'path': '/driver_info/ipmi_username',
'value': new_username},
{'op': 'add', 'path': '/driver_info/ipmi_password',
'value': new_password}]
if not utils.get_ipmi_address(node) and node_info.get('ipmi_address'):
patch.append({'op': 'add', 'path': '/driver_info/ipmi_address',
'value': node_info['ipmi_address']})
utils.retry_on_conflict(ironic.node.update, cached_node.uuid, patch)
for attempt in range(_CREDENTIALS_WAIT_RETRIES):
try:
# We use this call because it requires valid credentials.
# We don't care about boot device, obviously.
ironic.node.get_boot_device(cached_node.uuid)
except Exception as exc:
LOG.info(_LI('Waiting for credentials update on node %(node)s,'
' attempt %(attempt)d current error is %(exc)s') %
{'node': cached_node.uuid,
'attempt': attempt, 'exc': exc})
eventlet.greenthread.sleep(_CREDENTIALS_WAIT_PERIOD)
else:
_finish(ironic, cached_node)
return
msg = (_('Failed to validate updated IPMI credentials for node '
'%s, node might require maintenance') % cached_node.uuid)
cached_node.finished(error=msg)
raise utils.Error(msg)
def _prepare_for_pxe(ironic, cached_node, macs, setup_ipmi_credentials):
if macs:
LOG.info('Whitelisting MAC\'s %s for node %s on the firewall',
macs, cached_node.uuid)
firewall.update_filters(ironic)
if not setup_ipmi_credentials:
try:
utils.retry_on_conflict(ironic.node.set_boot_device,
cached_node.uuid, 'pxe', persistent=False)
except Exception as exc:
LOG.warning('Failed to set boot device to PXE for node %s: %s',
cached_node.uuid, exc)
try:
utils.retry_on_conflict(ironic.node.set_power_state,
cached_node.uuid, 'reboot')
except Exception as exc:
raise utils.Error('Failed to power on node %s, check it\'s power '
'management configuration:\n%s'
% (cached_node.uuid, exc))
else:
LOG.info('Introspection environment is ready for node %s, '
'manual power on is required within %d seconds',
cached_node.uuid, conf.getint('discoverd', 'timeout'))
def _prepare_for_pxe(ironic, cached_node, macs, setup_ipmi_credentials):
if macs:
LOG.info('Whitelisting MAC\'s %s for node %s on the firewall',
macs, cached_node.uuid)
firewall.update_filters(ironic)
if not setup_ipmi_credentials:
try:
utils.retry_on_conflict(ironic.node.set_boot_device,
cached_node.uuid, 'pxe', persistent=False)
except Exception as exc:
LOG.warning('Failed to set boot device to PXE for node %s: %s',
cached_node.uuid, exc)
try:
utils.retry_on_conflict(ironic.node.set_power_state,
cached_node.uuid, 'reboot')
except Exception as exc:
raise utils.Error('Failed to power on node %s, check it\'s power '
'management configuration:\n%s'
% (cached_node.uuid, exc))
else:
LOG.info('Introspection environment is ready for node %s, '
'manual power on is required within %d seconds',
cached_node.uuid, conf.getint('discoverd', 'timeout'))
def _finish(ironic, cached_node):
_force_power_off(ironic, cached_node)
patch = [{
'op': 'add',
'path': '/extra/newly_discovered',
'value': 'true'
}, {
'op': 'remove',
'path': '/extra/on_discovery'
}]
utils.retry_on_conflict(ironic.node.update, cached_node.uuid, patch)
cached_node.finished()
LOG.info('Introspection finished successfully for node %s',
cached_node.uuid)
def test_retry_on_conflict(self):
call = mock.Mock()
call.side_effect = ([exceptions.Conflict()] * (utils.RETRY_COUNT - 1)
+ [mock.sentinel.result])
res = utils.retry_on_conflict(call, 1, 2, x=3)
self.assertEqual(mock.sentinel.result, res)
call.assert_called_with(1, 2, x=3)
self.assertEqual(utils.RETRY_COUNT, call.call_count)
def test_retry_on_conflict(self):
call = mock.Mock()
call.side_effect = ([exceptions.Conflict()] * (utils.RETRY_COUNT - 1)
+ [mock.sentinel.result])
res = utils.retry_on_conflict(call, 1, 2, x=3)
self.assertEqual(mock.sentinel.result, res)
call.assert_called_with(1, 2, x=3)
self.assertEqual(utils.RETRY_COUNT, call.call_count)
def _background_start_discover(ironic, node, setup_ipmi_credentials):
patch = [{'op': 'add', 'path': '/extra/on_discovery', 'value': 'true'}]
utils.retry_on_conflict(ironic.node.update, node.uuid, patch)
# TODO(dtantsur): pagination
macs = [p.address for p in ironic.node.list_ports(node.uuid, limit=0)]
cached_node = node_cache.add_node(node.uuid,
bmc_address=_get_ipmi_address(node),
mac=macs)
cached_node.set_option('setup_ipmi_credentials', setup_ipmi_credentials)
try:
_prepare_for_pxe(ironic, cached_node, macs, setup_ipmi_credentials)
except utils.Error as exc:
cached_node.finished(error=str(exc))
except Exception as exc:
msg = 'Unexpected exception during preparing for PXE boot'
LOG.exception(msg)
cached_node.finished(error=msg)
def _background_start_discover(ironic, node, setup_ipmi_credentials):
patch = [{'op': 'add', 'path': '/extra/on_discovery', 'value': 'true'}]
utils.retry_on_conflict(ironic.node.update, node.uuid, patch)
# TODO(dtantsur): pagination
macs = [p.address for p in ironic.node.list_ports(node.uuid, limit=0)]
cached_node = node_cache.add_node(
node.uuid,
bmc_address=node.driver_info.get('ipmi_address'),
mac=macs)
cached_node.set_option('setup_ipmi_credentials', setup_ipmi_credentials)
try:
_prepare_for_pxe(ironic, cached_node, macs, setup_ipmi_credentials)
except utils.Error as exc:
cached_node.finished(error=str(exc))
except Exception as exc:
msg = 'Unexpected exception during preparing for PXE boot'
LOG.exception(msg)
cached_node.finished(error=msg)
def _finish(ironic, cached_node):
LOG.debug('Forcing power off of node %s', cached_node.uuid)
try:
utils.retry_on_conflict(ironic.node.set_power_state,
cached_node.uuid, 'off')
except Exception as exc:
msg = (_('Failed to power off node %(node)s, check it\'s power '
'management configuration: %(exc)s') %
{'node': cached_node.uuid, 'exc': exc})
cached_node.finished(error=msg)
raise utils.Error(msg)
cached_node.finished()
patch = [{'op': 'add', 'path': '/extra/newly_discovered', 'value': 'true'},
{'op': 'remove', 'path': '/extra/on_discovery'}]
utils.retry_on_conflict(ironic.node.update, cached_node.uuid, patch)
LOG.info(_LI('Introspection finished successfully for node %s'),
cached_node.uuid)
def _process_node(ironic, node, node_info, cached_node):
# NOTE(dtantsur): repeat the check in case something changed
utils.check_provision_state(node)
ports = {}
for mac in (node_info.get('macs') or ()):
try:
port = ironic.port.create(node_uuid=node.uuid, address=mac)
ports[mac] = port
except exceptions.Conflict:
LOG.warning(_LW('MAC %(mac)s appeared in introspection data for '
'node %(node)s, but already exists in '
'database - skipping') %
{'mac': mac, 'node': node.uuid})
node_patches, port_patches = _run_post_hooks(node, ports, node_info)
# Invalidate cache in case of hooks modifying options
cached_node.invalidate_cache()
node = utils.retry_on_conflict(ironic.node.update, node.uuid, node_patches)
for mac, patches in port_patches.items():
utils.retry_on_conflict(ironic.port.update, ports[mac].uuid, patches)
LOG.debug('Node %s was updated with data from introspection process, '
'patches %s, port patches %s',
node.uuid, node_patches, port_patches)
firewall.update_filters(ironic)
if cached_node.options.get('new_ipmi_credentials'):
new_username, new_password = (
cached_node.options.get('new_ipmi_credentials'))
eventlet.greenthread.spawn_n(_finish_set_ipmi_credentials,
ironic, node, cached_node, node_info,
new_username, new_password)
return {'ipmi_setup_credentials': True,
'ipmi_username': new_username,
'ipmi_password': new_password}
else:
eventlet.greenthread.spawn_n(_finish, ironic, cached_node)
return {}
def _process_node(ironic, node, node_info, cached_node):
ports = {}
for mac in (node_info.get('macs') or ()):
try:
port = ironic.port.create(node_uuid=node.uuid, address=mac)
ports[mac] = port
except exceptions.Conflict:
LOG.warning(
'MAC %(mac)s appeared in introspection data for '
'node %(node)s, but already exists in '
'database - skipping', {
'mac': mac,
'node': node.uuid
})
node_patches, port_patches = _run_post_hooks(node, ports, node_info)
node = utils.retry_on_conflict(ironic.node.update, node.uuid, node_patches)
for mac, patches in port_patches.items():
utils.retry_on_conflict(ironic.port.update, ports[mac].uuid, patches)
LOG.debug(
'Node %s was updated with data from introspection process, '
'patches %s, port patches %s', node.uuid, node_patches, port_patches)
firewall.update_filters(ironic)
if cached_node.options.get('setup_ipmi_credentials'):
eventlet.greenthread.spawn_n(_wait_for_power_management, ironic,
cached_node)
return {
'ipmi_setup_credentials': True,
'ipmi_username': node.driver_info.get('ipmi_username'),
'ipmi_password': node.driver_info.get('ipmi_password')
}
else:
eventlet.greenthread.spawn_n(_finish, ironic, cached_node)
return {}
def _force_power_off(ironic, cached_node):
LOG.debug('Forcing power off of node %s', cached_node.uuid)
try:
utils.retry_on_conflict(ironic.node.set_power_state, cached_node.uuid,
'off')
except Exception as exc:
msg = ('Failed to power off node %s, check it\'s power '
'management configuration: %s' % (cached_node.uuid, exc))
cached_node.finished(error=msg)
raise utils.Error(msg)
deadline = cached_node.started_at + conf.getint('discoverd', 'timeout')
while time.time() < deadline:
node = ironic.node.get(cached_node.uuid)
if (node.power_state or '').lower() == 'power off':
return
LOG.info('Waiting for node %s to power off, current state is %s',
cached_node.uuid, node.power_state)
eventlet.greenthread.sleep(_POWER_OFF_CHECK_PERIOD)
msg = ('Timeout waiting for node %s to power off after introspection' %
cached_node.uuid)
cached_node.finished(error=msg)
raise utils.Error(msg)
def introspect(uuid, new_ipmi_credentials=None):
"""Initiate hardware properties introspection for a given node.
:param uuid: node uuid
:param new_ipmi_credentials: tuple (new username, new password) or None
:raises: Error
"""
ironic = utils.get_client()
try:
node = ironic.node.get(uuid)
except exceptions.NotFound:
raise utils.Error(_("Cannot find node %s") % uuid, code=404)
except exceptions.HttpError as exc:
raise utils.Error(_("Cannot get node %(node)s: %(exc)s") %
{'node': uuid, 'exc': exc})
utils.check_provision_state(node)
if new_ipmi_credentials:
new_ipmi_credentials = (
_validate_ipmi_credentials(node, new_ipmi_credentials))
else:
validation = utils.retry_on_conflict(ironic.node.validate, node.uuid)
if not validation.power['result']:
msg = _('Failed validation of power interface for node %(node)s, '
'reason: %(reason)s')
raise utils.Error(msg % {'node': node.uuid,
'reason': validation.power['reason']})
cached_node = node_cache.add_node(node.uuid,
bmc_address=utils.get_ipmi_address(node))
cached_node.set_option('new_ipmi_credentials', new_ipmi_credentials)
def _handle_exceptions():
try:
_background_introspect(ironic, cached_node)
except utils.Error as exc:
cached_node.finished(error=str(exc))
except Exception as exc:
msg = _('Unexpected exception in background introspection thread')
LOG.exception(msg)
cached_node.finished(error=msg)
eventlet.greenthread.spawn_n(_handle_exceptions)
def introspect(uuid, setup_ipmi_credentials=False):
"""Initiate hardware properties introspection for a given node.
:param uuid: node uuid
:raises: Error
"""
ironic = utils.get_client()
try:
node = ironic.node.get(uuid)
except exceptions.NotFound:
raise utils.Error("Cannot find node %s" % uuid, code=404)
except exceptions.HttpError as exc:
raise utils.Error("Cannot get node %s: %s" % (uuid, exc))
if (setup_ipmi_credentials and not
conf.getboolean('discoverd', 'enable_setting_ipmi_credentials')):
raise utils.Error(
'IPMI credentials setup is disabled in configuration')
if not node.maintenance:
provision_state = node.provision_state
if provision_state and provision_state.lower() not in VALID_STATES:
msg = ('Refusing to introspect node %s with provision state "%s" '
'and maintenance mode off')
raise utils.Error(msg % (node.uuid, provision_state))
power_state = node.power_state
if power_state and power_state.lower() not in VALID_POWER_STATES:
msg = ('Refusing to introspect node %s with power state "%s" '
'and maintenance mode off')
raise utils.Error(msg % (node.uuid, power_state))
else:
LOG.info('Node %s is in maintenance mode, skipping power and provision'
' states check')
if not setup_ipmi_credentials:
validation = utils.retry_on_conflict(ironic.node.validate, node.uuid)
if not validation.power['result']:
msg = ('Failed validation of power interface for node %s, '
'reason: %s')
raise utils.Error(msg % (node.uuid, validation.power['reason']))
eventlet.greenthread.spawn_n(_background_start_discover, ironic, node,
setup_ipmi_credentials=setup_ipmi_credentials)
def _wait_for_power_management(ironic, cached_node):
deadline = cached_node.started_at + conf.getint('discoverd', 'timeout')
while time.time() < deadline:
eventlet.greenthread.sleep(_POWER_CHECK_PERIOD)
validation = utils.retry_on_conflict(ironic.node.validate,
cached_node.uuid)
if validation.power['result']:
_finish(ironic, cached_node)
return
LOG.debug(
'Waiting for management credentials on node %s '
'to be updated, current error: %s', cached_node.uuid,
validation.power['reason'])
msg = ('Timeout waiting for power credentials update of node %s '
'after introspection' % cached_node.uuid)
LOG.error(msg)
cached_node.finished(error=msg)
def introspect(uuid, setup_ipmi_credentials=False):
"""Initiate hardware properties introspection for a given node.
:param uuid: node uuid
:raises: Error
"""
ironic = utils.get_client()
try:
node = ironic.node.get(uuid)
except exceptions.NotFound:
raise utils.Error("Cannot find node %s" % uuid, code=404)
except exceptions.HttpError as exc:
raise utils.Error("Cannot get node %s: %s" % (uuid, exc))
if (setup_ipmi_credentials and not
conf.getboolean('discoverd', 'enable_setting_ipmi_credentials')):
raise utils.Error(
'IPMI credentials setup is disabled in configuration')
if not node.maintenance:
provision_state = node.provision_state
if provision_state and provision_state.lower() not in VALID_STATES:
msg = ('Refusing to introspect node %s with provision state "%s" '
'and maintenance mode off')
raise utils.Error(msg % (node.uuid, provision_state))
else:
LOG.info('Node %s is in maintenance mode, skipping power and provision'
' states check')
if not setup_ipmi_credentials:
validation = utils.retry_on_conflict(ironic.node.validate, node.uuid)
if not validation.power['result']:
msg = ('Failed validation of power interface for node %s, '
'reason: %s')
raise utils.Error(msg % (node.uuid, validation.power['reason']))
eventlet.greenthread.spawn_n(_background_start_discover, ironic, node,
setup_ipmi_credentials=setup_ipmi_credentials)
def _background_introspect(ironic, cached_node):
patch = [{'op': 'add', 'path': '/extra/on_discovery', 'value': 'true'}]
utils.retry_on_conflict(ironic.node.update, cached_node.uuid, patch)
# TODO(dtantsur): pagination
macs = [p.address for p in ironic.node.list_ports(cached_node.uuid,
limit=0)]
if macs:
cached_node.add_attribute(node_cache.MACS_ATTRIBUTE, macs)
LOG.info(_LI('Whitelisting MAC\'s %(macs)s for node %(node)s on the'
' firewall') %
{'macs': macs, 'node': cached_node.uuid})
firewall.update_filters(ironic)
if not cached_node.options.get('new_ipmi_credentials'):
try:
utils.retry_on_conflict(ironic.node.set_boot_device,
cached_node.uuid, 'pxe', persistent=False)
except Exception as exc:
LOG.warning(_LW('Failed to set boot device to PXE for'
' node %(node)s: %(exc)s') %
{'node': cached_node.uuid, 'exc': exc})
try:
utils.retry_on_conflict(ironic.node.set_power_state,
cached_node.uuid, 'reboot')
except Exception as exc:
raise utils.Error(_('Failed to power on node %(node)s,'
' check it\'s power '
'management configuration:\n%(exc)s')
% {'node': cached_node.uuid, 'exc': exc})
else:
LOG.info(_LI('Introspection environment is ready for node %(node)s, '
'manual power on is required within %(timeout)d seconds') %
{'node': cached_node.uuid,
'timeout': CONF.discoverd.timeout})