def _abort(node_info, ironic):
# runs in background
if node_info.finished_at is not None:
# introspection already finished; nothing to do
LOG.info(_LI('Cannot abort introspection as it is already '
'finished'), node_info=node_info)
node_info.release_lock()
return
# block this node from PXE Booting the introspection image
try:
firewall.update_filters(ironic)
except Exception as exc:
# Note(mkovacik): this will be retried in firewall update
# periodic task; we continue aborting
LOG.warning(_LW('Failed to update firewall filters: %s'), exc,
node_info=node_info)
# finish the introspection
LOG.debug('Forcing power-off', node_info=node_info)
try:
ironic.node.set_power_state(node_info.uuid, 'off')
except Exception as exc:
LOG.warning(_LW('Failed to power off node: %s'), exc,
node_info=node_info)
node_info.finished(error=_('Canceled by operator'))
LOG.info(_LI('Introspection aborted'), node_info=node_info)
def _finish_common(node_info, ironic, introspection_data, power_off=True):
if power_off:
LOG.debug('Forcing power off of node %s', node_info.uuid)
try:
ironic.node.set_power_state(node_info.uuid, 'off')
except Exception as exc:
if node_info.node().provision_state == 'enroll':
LOG.info(_LI("Failed to power off the node in"
"'enroll' state, ignoring; error was "
"%s"), exc, node_info=node_info,
data=introspection_data)
else:
msg = (_('Failed to power off node %(node)s, check '
'its power management configuration: '
'%(exc)s') % {'node': node_info.uuid, 'exc':
exc})
node_info.finished(error=msg)
raise utils.Error(msg, node_info=node_info,
data=introspection_data)
LOG.info(_LI('Node powered-off'), node_info=node_info,
data=introspection_data)
node_info.finished()
LOG.info(_LI('Introspection finished successfully'),
node_info=node_info, data=introspection_data)
def _abort(node_info, ironic):
# runs in background
if node_info.finished_at is not None:
# introspection already finished; nothing to do
LOG.info(_LI('Cannot abort introspection as it is already '
'finished'),
node_info=node_info)
node_info.release_lock()
return
# block this node from PXE Booting the introspection image
try:
firewall.update_filters(ironic)
except Exception as exc:
# Note(mkovacik): this will be retried in firewall update
# periodic task; we continue aborting
LOG.warning(_LW('Failed to update firewall filters: %s'),
exc,
node_info=node_info)
# finish the introspection
LOG.debug('Forcing power-off', node_info=node_info)
try:
ironic.node.set_power_state(node_info.uuid, 'off')
except Exception as exc:
LOG.warning(_LW('Failed to power off node: %s'),
exc,
node_info=node_info)
node_info.finished(error=_('Canceled by operator'))
LOG.info(_LI('Introspection aborted'), node_info=node_info)
def _finish(ironic, node_info, introspection_data, power_off=True):
if power_off:
LOG.debug('Forcing power off of node %s', node_info.uuid)
try:
ironic.node.set_power_state(node_info.uuid, 'off')
except Exception as exc:
if node_info.node().provision_state == 'enroll':
LOG.info(_LI("Failed to power off the node in"
"'enroll' state, ignoring; error was "
"%s") % exc, node_info=node_info,
data=introspection_data)
else:
msg = (_('Failed to power off node %(node)s, check '
'its power management configuration: '
'%(exc)s') % {'node': node_info.uuid, 'exc':
exc})
node_info.finished(error=msg)
raise utils.Error(msg, node_info=node_info,
data=introspection_data)
LOG.info(_LI('Node powered-off'), node_info=node_info,
data=introspection_data)
node_info.finished()
LOG.info(_LI('Introspection finished successfully'),
node_info=node_info, data=introspection_data)
def _background_introspect_locked(ironic, node_info):
# TODO(dtantsur): pagination
macs = list(node_info.ports())
if macs:
node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs)
LOG.info(
_LI('Whitelisting MAC\'s %(macs)s for node %(node)s on the'
' firewall') % {
'macs': macs,
'node': node_info.uuid
})
firewall.update_filters(ironic)
attrs = node_info.attributes
if CONF.processing.node_not_found_hook is None and not attrs:
raise utils.Error(
_('No lookup attributes were found for node %s, inspector won\'t '
'be able to find it after introspection. Consider creating '
'ironic ports or providing an IPMI address.') % node_info.uuid)
LOG.info(
_LI('The following attributes will be used for looking up '
'node %(uuid)s: %(attrs)s'), {
'attrs': attrs,
'uuid': node_info.uuid
})
if not node_info.options.get('new_ipmi_credentials'):
try:
ironic.node.set_boot_device(node_info.uuid,
'pxe',
persistent=False)
except Exception as exc:
LOG.warning(
_LW('Failed to set boot device to PXE for'
' node %(node)s: %(exc)s') % {
'node': node_info.uuid,
'exc': exc
})
try:
ironic.node.set_power_state(node_info.uuid, 'reboot')
except Exception as exc:
raise utils.Error(
_('Failed to power on node %(node)s,'
' check it\'s power '
'management configuration:\n%(exc)s') % {
'node': node_info.uuid,
'exc': exc
})
LOG.info(_LI('Introspection started successfully for node %s'),
node_info.uuid)
else:
LOG.info(
_LI('Introspection environment is ready for node %(node)s, '
'manual power on is required within %(timeout)d seconds') % {
'node': node_info.uuid,
'timeout': CONF.timeout
})
def _background_introspect(ironic, node_info):
global _LAST_INTROSPECTION_TIME
# TODO(dtantsur): pagination
macs = list(node_info.ports())
if macs:
node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs)
LOG.info(_LI('Whitelisting MAC\'s %(macs)s for node %(node)s on the'
' firewall') %
{'macs': macs, 'node': node_info.uuid})
firewall.update_filters(ironic)
attrs = node_info.attributes
if CONF.processing.node_not_found_hook is None and not attrs:
raise utils.Error(
_('No lookup attributes were found for node %s, inspector won\'t '
'be able to find it after introspection. Consider creating '
'ironic ports or providing an IPMI address.') % node_info.uuid)
LOG.info(_LI('The following attributes will be used for looking up '
'node %(uuid)s: %(attrs)s'),
{'attrs': attrs, 'uuid': node_info.uuid})
if not node_info.options.get('new_ipmi_credentials'):
try:
ironic.node.set_boot_device(node_info.uuid, 'pxe',
persistent=False)
except Exception as exc:
LOG.warning(_LW('Failed to set boot device to PXE for'
' node %(node)s: %(exc)s') %
{'node': node_info.uuid, 'exc': exc})
if re.match(CONF.introspection_delay_drivers, node_info.node().driver):
LOG.debug('Attempting to acquire lock on last introspection time')
with _LAST_INTROSPECTION_LOCK:
delay = (_LAST_INTROSPECTION_TIME - time.time()
+ CONF.introspection_delay)
if delay > 0:
LOG.debug('Waiting %d seconds before sending the next '
'node on introspection', delay)
time.sleep(delay)
_LAST_INTROSPECTION_TIME = time.time()
try:
ironic.node.set_power_state(node_info.uuid, 'reboot')
except Exception as exc:
raise utils.Error(_('Failed to power on node %(node)s,'
' check it\'s power '
'management configuration:\n%(exc)s')
% {'node': node_info.uuid, 'exc': exc})
LOG.info(_LI('Introspection started successfully for node %s'),
node_info.uuid)
else:
LOG.info(_LI('Introspection environment is ready for node %(node)s, '
'manual power on is required within %(timeout)d seconds') %
{'node': node_info.uuid,
'timeout': CONF.timeout})
def check_conditions(self, node_info, data):
"""Check if conditions are true for a given node.
:param node_info: a NodeInfo object
:param data: introspection data
:returns: True if conditions match, otherwise False
"""
LOG.debug('Checking rule "%s"', self.description,
node_info=node_info, data=data)
ext_mgr = plugins_base.rule_conditions_manager()
for cond in self._conditions:
scheme, path = _parse_path(cond.field)
if scheme == 'node':
source_data = node_info.node().to_dict()
elif scheme == 'data':
source_data = data
field_values = jsonpath.parse(path).find(source_data)
field_values = [x.value for x in field_values]
cond_ext = ext_mgr[cond.op].obj
if not field_values:
if cond_ext.ALLOW_NONE:
LOG.debug('Field with JSON path %s was not found in data',
cond.field, node_info=node_info, data=data)
field_values = [None]
else:
LOG.info(_LI('Field with JSON path %(path)s was not found '
'in data, rule "%(rule)s" will not '
'be applied'),
{'path': cond.field, 'rule': self.description},
node_info=node_info, data=data)
return False
for value in field_values:
result = cond_ext.check(node_info, value, cond.params)
if cond.invert:
result = not result
if (cond.multiple == 'first'
or (cond.multiple == 'all' and not result)
or (cond.multiple == 'any' and result)):
break
if not result:
LOG.info(_LI('Rule "%(rule)s" will not be applied: condition '
'%(field)s %(op)s %(params)s failed'),
{'rule': self.description, 'field': cond.field,
'op': cond.op, 'params': cond.params},
node_info=node_info, data=data)
return False
LOG.info(_LI('Rule "%s" will be applied'), self.description,
node_info=node_info, data=data)
return True
def before_processing(self, introspection_data, **kwargs):
"""Validate information about network interfaces."""
bmc_address = introspection_data.get('ipmi_address')
if not introspection_data.get('interfaces'):
raise utils.Error(_('No interfaces supplied by the ramdisk'))
valid_interfaces = {
n: iface for n, iface in introspection_data['interfaces'].items()
if utils.is_valid_mac(iface.get('mac'))
}
pxe_mac = introspection_data.get('boot_interface')
if CONF.processing.add_ports == 'pxe' and not pxe_mac:
LOG.warning(_LW('No boot interface provided in the introspection '
'data, will add all ports with IP addresses'))
if CONF.processing.add_ports == 'pxe' and pxe_mac:
LOG.info(_LI('PXE boot interface was %s'), pxe_mac)
if '-' in pxe_mac:
# pxelinux format: 01-aa-bb-cc-dd-ee-ff
pxe_mac = pxe_mac.split('-', 1)[1]
pxe_mac = pxe_mac.replace('-', ':').lower()
valid_interfaces = {
n: iface for n, iface in valid_interfaces.items()
if iface['mac'].lower() == pxe_mac
}
elif CONF.processing.add_ports != 'all':
valid_interfaces = {
n: iface for n, iface in valid_interfaces.items()
if iface.get('ip')
}
if not valid_interfaces:
raise utils.Error(_('No valid interfaces found for node with '
'BMC %(ipmi_address)s, got %(interfaces)s') %
{'ipmi_address': bmc_address,
'interfaces': introspection_data['interfaces']})
elif valid_interfaces != introspection_data['interfaces']:
invalid = {n: iface
for n, iface in introspection_data['interfaces'].items()
if n not in valid_interfaces}
LOG.warning(_LW(
'The following interfaces were invalid or not eligible in '
'introspection data for node with BMC %(ipmi_address)s and '
'were excluded: %(invalid)s'),
{'invalid': invalid, 'ipmi_address': bmc_address})
LOG.info(_LI('Eligible interfaces are %s'), valid_interfaces)
introspection_data['all_interfaces'] = introspection_data['interfaces']
introspection_data['interfaces'] = valid_interfaces
valid_macs = [iface['mac'] for iface in valid_interfaces.values()]
introspection_data['macs'] = valid_macs
def before_processing(self, introspection_data, **kwargs):
"""Validate information about network interfaces."""
bmc_address = introspection_data.get('ipmi_address')
if not introspection_data.get('interfaces'):
raise utils.Error(_('No interfaces supplied by the ramdisk'))
valid_interfaces = {
n: iface for n, iface in introspection_data['interfaces'].items()
if utils.is_valid_mac(iface.get('mac'))
}
pxe_mac = introspection_data.get('boot_interface')
if CONF.processing.add_ports == 'pxe' and not pxe_mac:
LOG.warning(_LW('No boot interface provided in the introspection '
'data, will add all ports with IP addresses'))
if CONF.processing.add_ports == 'pxe' and pxe_mac:
LOG.info(_LI('PXE boot interface was %s'), pxe_mac)
if '-' in pxe_mac:
# pxelinux format: 01-aa-bb-cc-dd-ee-ff
pxe_mac = pxe_mac.split('-', 1)[1]
pxe_mac = pxe_mac.replace('-', ':').lower()
valid_interfaces = {
n: iface for n, iface in valid_interfaces.items()
if iface['mac'].lower() == pxe_mac
}
elif CONF.processing.add_ports != 'all':
valid_interfaces = {
n: iface for n, iface in valid_interfaces.items()
if iface.get('ip')
}
if not valid_interfaces:
raise utils.Error(_('No valid interfaces found for node with '
'BMC %(ipmi_address)s, got %(interfaces)s') %
{'ipmi_address': bmc_address,
'interfaces': introspection_data['interfaces']})
elif valid_interfaces != introspection_data['interfaces']:
invalid = {n: iface
for n, iface in introspection_data['interfaces'].items()
if n not in valid_interfaces}
LOG.warning(_LW(
'The following interfaces were invalid or not eligible in '
'introspection data for node with BMC %(ipmi_address)s and '
'were excluded: %(invalid)s'),
{'invalid': invalid, 'ipmi_address': bmc_address})
LOG.info(_LI('Eligible interfaces are %s'), valid_interfaces)
introspection_data['all_interfaces'] = introspection_data['interfaces']
introspection_data['interfaces'] = valid_interfaces
valid_macs = [iface['mac'] for iface in valid_interfaces.values()]
introspection_data['macs'] = valid_macs
def check_conditions(self, node_info, data):
"""Check if conditions are true for a given node.
:param node_info: a NodeInfo object
:param data: introspection data
:returns: True if conditions match, otherwise False
"""
LOG.debug('Checking rule "%(descr)s" on node %(uuid)s',
{'descr': self.description, 'uuid': node_info.uuid})
ext_mgr = plugins_base.rule_conditions_manager()
for cond in self._conditions:
field_values = jsonpath.parse(cond.field).find(data)
field_values = [x.value for x in field_values]
cond_ext = ext_mgr[cond.op].obj
if not field_values:
if cond_ext.ALLOW_NONE:
LOG.debug('Field with JSON path %(path)s was not found in '
'data for node %(uuid)s',
{'path': cond.field, 'uuid': node_info.uuid})
field_values = [None]
else:
LOG.info(_LI('Field with JSON path %(path)s was not found '
'in data for node %(uuid)s, rule "%(rule)s" '
'will not be applied'),
{'path': cond.field, 'uuid': node_info.uuid,
'rule': self.description})
return False
for value in field_values:
result = cond_ext.check(node_info, value, cond.params)
if (cond.multiple == 'first'
or (cond.multiple == 'all' and not result)
or (cond.multiple == 'any' and result)):
break
if not result:
LOG.info(_LI('Rule "%(rule)s" will not be applied to node '
'%(uuid)s: condition %(field)s %(op)s %(params)s '
'failed'),
{'rule': self.description, 'uuid': node_info.uuid,
'field': cond.field, 'op': cond.op,
'params': cond.params})
return False
LOG.info(_LI('Rule "%(rule)s" will be applied to node %(uuid)s'),
{'rule': self.description, 'uuid': node_info.uuid})
return True
def _finish_set_ipmi_credentials(ironic, node, node_info, introspection_data,
new_username, new_password):
patch = [{'op': 'add', 'path': '/driver_info/ipmi_username',
'value': new_username},
{'op': 'add', 'path': '/driver_info/ipmi_password',
'value': new_password}]
if (not ir_utils.get_ipmi_address(node) and
introspection_data.get('ipmi_address')):
patch.append({'op': 'add', 'path': '/driver_info/ipmi_address',
'value': introspection_data['ipmi_address']})
node_info.patch(patch)
for attempt in range(_CREDENTIALS_WAIT_RETRIES):
try:
# We use this call because it requires valid credentials.
# We don't care about boot device, obviously.
ironic.node.get_boot_device(node_info.uuid)
except Exception as exc:
LOG.info(_LI('Waiting for credentials update, attempt %(attempt)d '
'current error is %(exc)s') %
{'attempt': attempt, 'exc': exc},
node_info=node_info, data=introspection_data)
eventlet.greenthread.sleep(_CREDENTIALS_WAIT_PERIOD)
else:
_finish(ironic, node_info, introspection_data)
return
msg = (_('Failed to validate updated IPMI credentials for node '
'%s, node might require maintenance') % node_info.uuid)
node_info.finished(error=msg)
raise utils.Error(msg, node_info=node_info, data=introspection_data)
def before_processing(self, introspection_data, **kwargs):
"""Validate information about network interfaces."""
bmc_address = utils.get_ipmi_address_from_data(introspection_data)
if bmc_address:
introspection_data['ipmi_address'] = bmc_address
else:
LOG.debug(
'No BMC address provided in introspection data, '
'assuming virtual environment',
data=introspection_data)
all_interfaces = self._get_interfaces(introspection_data)
interfaces = self._validate_interfaces(all_interfaces,
introspection_data)
LOG.info(_LI('Using network interface(s): %s'),
', '.join('%s %s' % (name, items)
for (name, items) in interfaces.items()),
data=introspection_data)
introspection_data['all_interfaces'] = all_interfaces
introspection_data['interfaces'] = interfaces
valid_macs = [iface['mac'] for iface in interfaces.values()]
introspection_data['macs'] = valid_macs
def before_update(self, introspection_data, node_info, **kwargs):
"""Drop ports that are not present in the data."""
if CONF.processing.keep_ports == 'present':
expected_macs = {
iface['mac']
for iface in introspection_data['all_interfaces'].values()
}
elif CONF.processing.keep_ports == 'added':
expected_macs = set(introspection_data['macs'])
else:
return
# list is required as we modify underlying dict
for port in list(node_info.ports().values()):
if port.address not in expected_macs:
LOG.info(
_LI("Deleting port %(port)s as its MAC %(mac)s is "
"not in expected MAC list %(expected)s for node "
"%(node)s"), {
'port': port.uuid,
'mac': port.address,
'expected': list(sorted(expected_macs)),
'node': node_info.uuid
})
node_info.delete_port(port)
def before_update(self, introspection_data, node_info, **kwargs):
"""Update node with scheduler properties."""
root_disk = introspection_data.get('root_disk')
if root_disk:
introspection_data['local_gb'] = root_disk['size'] // units.Gi
if CONF.processing.disk_partitioning_spacing:
introspection_data['local_gb'] -= 1
missing = [key for key in self.KEYS if not introspection_data.get(key)]
if missing:
raise utils.Error(
_('The following required parameters are missing: %s') %
missing)
LOG.info(
_LI('Discovered data: CPUs: %(cpus)s %(cpu_arch)s, '
'memory %(memory_mb)s MiB, disk %(local_gb)s GiB'),
{key: introspection_data.get(key)
for key in self.KEYS})
overwrite = CONF.processing.overwrite_existing
properties = {
key: str(introspection_data[key])
for key in self.KEYS
if overwrite or not node_info.node().properties.get(key)
}
node_info.update_properties(**properties)
def before_processing(self, introspection_data, **kwargs):
"""Adds fake local_gb value if it's missing from introspection_data."""
if not introspection_data.get('local_gb'):
LOG.info(_LI('No volume is found on the node. Adding a fake '
'value for "local_gb"'),
data=introspection_data)
introspection_data['local_gb'] = 1
def before_processing(self, introspection_data, **kwargs):
"""Adds fake local_gb value if it's missing from introspection_data."""
if not introspection_data.get('local_gb'):
LOG.info(_LI('No volume is found on the node. Adding a fake '
'value for "local_gb"'),
data=introspection_data)
introspection_data['local_gb'] = 1
def apply(node_info, data):
"""Apply rules to a node."""
rules = get_all()
if not rules:
LOG.debug('No custom introspection rules to apply to node %s',
node_info.uuid)
return
LOG.debug('Applying custom introspection rules to node %s', node_info.uuid)
to_rollback = []
to_apply = []
for rule in rules:
if rule.check_conditions(node_info, data):
to_apply.append(rule)
else:
to_rollback.append(rule)
if to_rollback:
LOG.debug('Running rollback actions on node %s', node_info.uuid)
for rule in to_rollback:
rule.apply_actions(node_info, rollback=True)
else:
LOG.debug('No rollback actions to apply on node %s', node_info.uuid)
if to_apply:
LOG.debug('Running actions on node %s', node_info.uuid)
for rule in to_apply:
rule.apply_actions(node_info, rollback=False)
else:
LOG.debug('No actions to apply on node %s', node_info.uuid)
LOG.info(_LI('Successfully applied custom introspection rules to node %s'),
node_info.uuid)
def before_update(self, introspection_data, node_info, **kwargs):
"""Update node with scheduler properties."""
inventory = introspection_data.get('inventory')
errors = []
root_disk = introspection_data.get('root_disk')
if root_disk:
introspection_data['local_gb'] = root_disk['size'] // units.Gi
if CONF.processing.disk_partitioning_spacing:
introspection_data['local_gb'] -= 1
elif inventory:
errors.append(_('root disk is not supplied by the ramdisk and '
'root_disk_selection hook is not enabled'))
if inventory:
try:
introspection_data['cpus'] = int(inventory['cpu']['count'])
introspection_data['cpu_arch'] = six.text_type(
inventory['cpu']['architecture'])
except (KeyError, ValueError, TypeError):
errors.append(_('malformed or missing CPU information: %s') %
inventory.get('cpu'))
try:
introspection_data['memory_mb'] = int(
inventory['memory']['physical_mb'])
except (KeyError, ValueError, TypeError):
errors.append(_('malformed or missing memory information: %s; '
'introspection requires physical memory size '
'from dmidecode') %
inventory.get('memory'))
else:
LOG.warning(_LW('No inventory provided: using old bash ramdisk '
'is deprecated, please switch to '
'ironic-python-agent'),
node_info=node_info, data=introspection_data)
missing = [key for key in self.KEYS
if not introspection_data.get(key)]
if missing:
raise utils.Error(
_('The following required parameters are missing: %s') %
missing,
node_info=node_info, data=introspection_data)
if errors:
raise utils.Error(_('The following problems encountered: %s') %
'; '.join(errors),
node_info=node_info, data=introspection_data)
LOG.info(_LI('Discovered data: CPUs: %(cpus)s %(cpu_arch)s, '
'memory %(memory_mb)s MiB, disk %(local_gb)s GiB'),
{key: introspection_data.get(key) for key in self.KEYS},
node_info=node_info, data=introspection_data)
overwrite = CONF.processing.overwrite_existing
properties = {key: str(introspection_data[key])
for key in self.KEYS if overwrite or
not node_info.node().properties.get(key)}
node_info.update_properties(**properties)
def _finish_set_ipmi_credentials(node_info, ironic, node, introspection_data,
new_username, new_password):
patch = [{'op': 'add', 'path': '/driver_info/ipmi_username',
'value': new_username},
{'op': 'add', 'path': '/driver_info/ipmi_password',
'value': new_password}]
new_ipmi_address = utils.get_ipmi_address_from_data(introspection_data)
if not ir_utils.get_ipmi_address(node) and new_ipmi_address:
patch.append({'op': 'add', 'path': '/driver_info/ipmi_address',
'value': new_ipmi_address})
node_info.patch(patch)
for attempt in range(_CREDENTIALS_WAIT_RETRIES):
try:
# We use this call because it requires valid credentials.
# We don't care about boot device, obviously.
ironic.node.get_boot_device(node_info.uuid)
except Exception as exc:
LOG.info(_LI('Waiting for credentials update, attempt %(attempt)d '
'current error is %(exc)s'),
{'attempt': attempt, 'exc': exc},
node_info=node_info, data=introspection_data)
eventlet.greenthread.sleep(_CREDENTIALS_WAIT_PERIOD)
else:
_finish_common(node_info, ironic, introspection_data)
return
msg = (_('Failed to validate updated IPMI credentials for node '
'%s, node might require maintenance') % node_info.uuid)
node_info.finished(error=msg)
raise utils.Error(msg, node_info=node_info, data=introspection_data)
def _reapply(node_info):
# runs in background
try:
introspection_data = _get_unprocessed_data(node_info.uuid)
except Exception as exc:
LOG.exception(_LE('Encountered exception while fetching '
'stored introspection data'),
node_info=node_info)
msg = (_('Unexpected exception %(exc_class)s while fetching '
'unprocessed introspection data from Swift: %(error)s') %
{'exc_class': exc.__class__.__name__, 'error': exc})
node_info.finished(error=msg)
return
try:
ironic = ir_utils.get_client()
except Exception as exc:
msg = _('Encountered an exception while getting the Ironic client: '
'%s') % exc
LOG.error(msg, node_info=node_info, data=introspection_data)
node_info.fsm_event(istate.Events.error)
node_info.finished(error=msg)
return
try:
_reapply_with_data(node_info, introspection_data)
except Exception as exc:
node_info.finished(error=str(exc))
return
_finish(node_info, ironic, introspection_data,
power_off=False)
LOG.info(_LI('Successfully reapplied introspection on stored '
'data'), node_info=node_info, data=introspection_data)
def init(self):
if utils.get_auth_strategy() != 'noauth':
utils.add_auth_middleware(app)
else:
LOG.warning(_LW('Starting unauthenticated, please check'
' configuration'))
if CONF.processing.store_data == 'none':
LOG.warning(_LW('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior'))
elif CONF.processing.store_data == 'swift':
LOG.info(_LI('Introspection data will be stored in Swift in the '
'container %s'), CONF.swift.container)
utils.add_cors_middleware(app)
db.init()
try:
hooks = [ext.name for ext in
plugins_base.processing_hooks_manager()]
except KeyError as exc:
# callback function raises MissingHookError derived from KeyError
# on missing hook
LOG.critical(_LC('Hook(s) %s failed to load or was not found'),
str(exc))
sys.exit(1)
LOG.info(_LI('Enabled processing hooks: %s'), hooks)
if CONF.firewall.manage_firewall:
firewall.init()
periodic_update_ = periodics.periodic(
spacing=CONF.firewall.firewall_update_period,
enabled=CONF.firewall.manage_firewall
)(periodic_update)
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period
)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(periodic_update_, None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()))
utils.executor().submit(self._periodics_worker.start)
def delete_all():
"""Delete all rules."""
with db.ensure_transaction() as session:
db.model_query(db.RuleAction, session=session).delete()
db.model_query(db.RuleCondition, session=session).delete()
db.model_query(db.Rule, session=session).delete()
LOG.info(_LI('All introspection rules were deleted'))
def _background_introspect_locked(ironic, node_info):
# TODO(dtantsur): pagination
macs = list(node_info.ports())
if macs:
node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs)
LOG.info(_LI('Whitelisting MAC\'s %s on the firewall'),
macs,
node_info=node_info)
firewall.update_filters(ironic)
attrs = node_info.attributes
if CONF.processing.node_not_found_hook is None and not attrs:
raise utils.Error(_(
'No lookup attributes were found, inspector won\'t '
'be able to find it after introspection, consider creating '
'ironic ports or providing an IPMI address'),
node_info=node_info)
LOG.info(_LI('The following attributes will be used for look up: %s'),
attrs,
node_info=node_info)
if not node_info.options.get('new_ipmi_credentials'):
try:
ironic.node.set_boot_device(node_info.uuid,
'pxe',
persistent=False)
except Exception as exc:
LOG.warning(_LW('Failed to set boot device to PXE: %s'),
exc,
node_info=node_info)
try:
ironic.node.set_power_state(node_info.uuid, 'reboot')
except Exception as exc:
raise utils.Error(_('Failed to power on the node, check it\'s '
'power management configuration: %s'),
exc,
node_info=node_info)
LOG.info(_LI('Introspection started successfully'),
node_info=node_info)
else:
LOG.info(_LI('Introspection environment is ready, manual power on is '
'required within %d seconds'),
CONF.timeout,
node_info=node_info)
def before_update(self, introspection_data, node_info, **kwargs):
current_devices = self._get_serials(introspection_data)
if not current_devices:
LOG.warning(_LW('No block device was received from ramdisk'),
node_info=node_info,
data=introspection_data)
return
node = node_info.node()
if 'root_device' in node.properties:
LOG.info(_LI('Root device is already known for the node'),
node_info=node_info,
data=introspection_data)
return
if 'block_devices' in node.extra:
# Compare previously discovered devices with the current ones
previous_devices = node.extra['block_devices']['serials']
new_devices = [
device for device in current_devices
if device not in previous_devices
]
if len(new_devices) > 1:
LOG.warning(_LW('Root device cannot be identified because '
'multiple new devices were found'),
node_info=node_info,
data=introspection_data)
return
elif len(new_devices) == 0:
LOG.warning(_LW('No new devices were found'),
node_info=node_info,
data=introspection_data)
return
node_info.patch([{
'op': 'remove',
'path': '/extra/block_devices'
}, {
'op': 'add',
'path': '/properties/root_device',
'value': {
'serial': new_devices[0]
}
}])
else:
# No previously discovered devices - save the inspector block
# devices in node.extra
node_info.patch([{
'op': 'add',
'path': '/extra/block_devices',
'value': {
'serials': current_devices
}
}])
def process(introspection_data):
"""Process data from the ramdisk.
This function heavily relies on the hooks to do the actual data processing.
"""
unprocessed_data = copy.deepcopy(introspection_data)
failures = []
_run_pre_hooks(introspection_data, failures)
node_info = _find_node_info(introspection_data, failures)
if node_info:
# Locking is already done in find_node() but may be not done in a
# node_not_found hook
node_info.acquire_lock()
if failures or node_info is None:
msg = _('The following failures happened during running '
'pre-processing hooks:\n%s') % '\n'.join(failures)
if node_info is not None:
node_info.finished(error='\n'.join(failures))
raise utils.Error(msg, node_info=node_info, data=introspection_data)
LOG.info(_LI('Matching node is %s'), node_info.uuid,
node_info=node_info, data=introspection_data)
if node_info.finished_at is not None:
# race condition or introspection canceled
raise utils.Error(_('Node processing already finished with '
'error: %s') % node_info.error,
node_info=node_info, code=400)
# Note(mkovacik): store data now when we're sure that a background
# thread won't race with other process() or introspect.abort()
# call
utils.executor().submit(_store_unprocessed_data, node_info,
unprocessed_data)
try:
node = node_info.node()
except exceptions.NotFound:
msg = _('Node was found in cache, but is not found in Ironic')
node_info.finished(error=msg)
raise utils.Error(msg, code=404, node_info=node_info,
data=introspection_data)
try:
return _process_node(node, introspection_data, node_info)
except utils.Error as exc:
node_info.finished(error=str(exc))
raise
except Exception as exc:
LOG.exception(_LE('Unexpected exception during processing'))
msg = _('Unexpected exception %(exc_class)s during processing: '
'%(error)s') % {'exc_class': exc.__class__.__name__,
'error': exc}
node_info.finished(error=msg)
raise utils.Error(msg, node_info=node_info, data=introspection_data)
def _detect_boot_mode(self, inventory, node_info, data=None):
boot_mode = inventory.get('boot', {}).get('current_boot_mode')
if boot_mode is not None:
LOG.info(_LI('Boot mode was %s'), boot_mode,
data=data, node_info=node_info)
return {'boot_mode': boot_mode}
else:
LOG.warning(_LW('No boot mode information available'),
data=data, node_info=node_info)
return {}
def init(self):
if utils.get_auth_strategy() != 'noauth':
utils.add_auth_middleware(app)
else:
LOG.warning(
_LW('Starting unauthenticated, please check'
' configuration'))
if CONF.processing.store_data == 'none':
LOG.warning(
_LW('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior'))
elif CONF.processing.store_data == 'swift':
LOG.info(
_LI('Introspection data will be stored in Swift in the '
'container %s'), CONF.swift.container)
utils.add_cors_middleware(app)
db.init()
try:
hooks = [
ext.name for ext in plugins_base.processing_hooks_manager()
]
except KeyError as exc:
# callback function raises MissingHookError derived from KeyError
# on missing hook
LOG.critical(_LC('Hook(s) %s failed to load or was not found'),
str(exc))
sys.exit(1)
LOG.info(_LI('Enabled processing hooks: %s'), hooks)
if CONF.firewall.manage_firewall:
firewall.init()
self._periodics_worker = periodics.PeriodicWorker(
callables=[(periodic_update, None, None),
(periodic_clean_up, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()))
utils.executor().submit(self._periodics_worker.start)
def before_update(self, introspection_data, node_info, **kwargs):
"""Update node with scheduler properties."""
inventory = utils.get_inventory(introspection_data,
node_info=node_info)
errors = []
root_disk = introspection_data.get('root_disk')
if root_disk:
introspection_data['local_gb'] = root_disk['size'] // units.Gi
if CONF.processing.disk_partitioning_spacing:
introspection_data['local_gb'] -= 1
else:
errors.append(
_('root disk is not supplied by the ramdisk and '
'root_disk_selection hook is not enabled'))
try:
introspection_data['cpus'] = int(inventory['cpu']['count'])
introspection_data['cpu_arch'] = six.text_type(
inventory['cpu']['architecture'])
except (KeyError, ValueError, TypeError):
errors.append(
_('malformed or missing CPU information: %s') %
inventory.get('cpu'))
try:
introspection_data['memory_mb'] = int(
inventory['memory']['physical_mb'])
except (KeyError, ValueError, TypeError):
errors.append(
_('malformed or missing memory information: %s; '
'introspection requires physical memory size '
'from dmidecode') % inventory.get('memory'))
if errors:
raise utils.Error(_('The following problems encountered: %s') %
'; '.join(errors),
node_info=node_info,
data=introspection_data)
LOG.info(_LI('Discovered data: CPUs: %(cpus)s %(cpu_arch)s, '
'memory %(memory_mb)s MiB, disk %(local_gb)s GiB'),
{key: introspection_data.get(key)
for key in self.KEYS},
node_info=node_info,
data=introspection_data)
overwrite = CONF.processing.overwrite_existing
properties = {
key: str(introspection_data[key])
for key in self.KEYS
if overwrite or not node_info.node().properties.get(key)
}
node_info.update_properties(**properties)
def _finish(ironic, node_info, introspection_data):
LOG.debug('Forcing power off of node %s', node_info.uuid)
try:
ironic.node.set_power_state(node_info.uuid, 'off')
except Exception as exc:
if node_info.node().provision_state == 'enroll':
LOG.info(_LI("Failed to power off the node in 'enroll' state, "
"ignoring; error was %s") % exc,
node_info=node_info, data=introspection_data)
else:
msg = (_('Failed to power off node %(node)s, check it\'s '
'power management configuration: %(exc)s') %
{'node': node_info.uuid, 'exc': exc})
node_info.finished(error=msg)
raise utils.Error(msg, node_info=node_info,
data=introspection_data)
node_info.finished()
LOG.info(_LI('Introspection finished successfully'),
node_info=node_info, data=introspection_data)
def _fsm_ctx(self):
fsm = self._get_fsm()
try:
yield fsm
finally:
if fsm.current_state != self.state:
LOG.info(_LI('Updating node state: %(current)s --> %(new)s'), {
'current': self.state,
'new': fsm.current_state
},
node_info=self)
self._set_state(fsm.current_state)
def _background_introspect_locked(ironic, node_info):
# TODO(dtantsur): pagination
macs = list(node_info.ports())
if macs:
node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs)
LOG.info(_LI('Whitelisting MAC\'s %s on the firewall'), macs,
node_info=node_info)
firewall.update_filters(ironic)
attrs = node_info.attributes
if CONF.processing.node_not_found_hook is None and not attrs:
raise utils.Error(
_('No lookup attributes were found, inspector won\'t '
'be able to find it after introspection, consider creating '
'ironic ports or providing an IPMI address'),
node_info=node_info)
LOG.info(_LI('The following attributes will be used for look up: %s'),
attrs, node_info=node_info)
if not node_info.options.get('new_ipmi_credentials'):
try:
ironic.node.set_boot_device(node_info.uuid, 'pxe',
persistent=False)
except Exception as exc:
LOG.warning(_LW('Failed to set boot device to PXE: %s'),
exc, node_info=node_info)
try:
ironic.node.set_power_state(node_info.uuid, 'reboot')
except Exception as exc:
raise utils.Error(_('Failed to power on the node, check it\'s '
'power management configuration: %s'),
exc, node_info=node_info)
LOG.info(_LI('Introspection started successfully'),
node_info=node_info)
else:
LOG.info(_LI('Introspection environment is ready, manual power on is '
'required within %d seconds'), CONF.timeout,
node_info=node_info)
def init():
if utils.get_auth_strategy() != 'noauth':
utils.add_auth_middleware(app)
else:
LOG.warning(
_LW('Starting unauthenticated, please check'
' configuration'))
if CONF.processing.store_data == 'none':
LOG.warning(
_LW('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not the '
'desired behavior'))
elif CONF.processing.store_data == 'swift':
LOG.info(
_LI('Introspection data will be stored in Swift in the '
'container %s'), CONF.swift.container)
db.init()
try:
hooks = [ext.name for ext in plugins_base.processing_hooks_manager()]
except KeyError as exc:
# stevedore raises KeyError on missing hook
LOG.critical(_LC('Hook %s failed to load or was not found'), str(exc))
sys.exit(1)
LOG.info(_LI('Enabled processing hooks: %s'), hooks)
if CONF.firewall.manage_firewall:
firewall.init()
period = CONF.firewall.firewall_update_period
utils.spawn_n(periodic_update, period)
if CONF.timeout > 0:
period = CONF.clean_up_period
utils.spawn_n(periodic_clean_up, period)
else:
LOG.warning(_LW('Timeout is disabled in configuration'))
def before_update(self, introspection_data, node_info, **kwargs):
if 'pci_devices' not in introspection_data:
if CONF.pci_devices.alias:
LOG.warning(_LW('No PCI devices information was received from '
'the ramdisk.'))
return
alias_count = {self.aliases[id_pair]: count for id_pair, count in
self._found_pci_devices_count(
introspection_data['pci_devices']).items()}
if alias_count:
node_info.update_capabilities(**alias_count)
LOG.info(_LI('Found the following PCI devices: %s'),
alias_count)
def delete(uuid):
"""Delete a rule by its UUID."""
with db.ensure_transaction() as session:
db.model_query(db.RuleAction,
session=session).filter_by(rule=uuid).delete()
db.model_query(db.RuleCondition,
session=session) .filter_by(rule=uuid).delete()
count = (db.model_query(db.Rule, session=session)
.filter_by(uuid=uuid).delete())
if not count:
raise utils.Error(_('Rule %s was not found') % uuid, code=404)
LOG.info(_LI('Introspection rule %s was deleted'), uuid)
def shutdown(self):
LOG.debug('Shutting down')
firewall.clean_up()
if self._periodics_worker is not None:
self._periodics_worker.stop()
self._periodics_worker.wait()
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
LOG.info(_LI('Shut down successfully'))
def shutdown(self):
LOG.debug('Shutting down')
firewall.clean_up()
if self._periodics_worker is not None:
self._periodics_worker.stop()
self._periodics_worker.wait()
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
LOG.info(_LI('Shut down successfully'))
def _finish(ironic, node_info):
LOG.debug('Forcing power off of node %s', node_info.uuid)
try:
ironic.node.set_power_state(node_info.uuid, 'off')
except Exception as exc:
msg = (_('Failed to power off node %(node)s, check it\'s power '
'management configuration: %(exc)s') %
{'node': node_info.uuid, 'exc': exc})
node_info.finished(error=msg)
raise utils.Error(msg)
node_info.finished()
LOG.info(_LI('Introspection finished successfully for node %s'),
node_info.uuid)
def before_update(self, introspection_data, node_info, **kwargs):
if 'pci_devices' not in introspection_data:
if CONF.pci_devices.alias:
LOG.warning(
_LW('No PCI devices information was received from '
'the ramdisk.'))
return
alias_count = {
self.aliases[id_pair]: count
for id_pair, count in self._found_pci_devices_count(
introspection_data['pci_devices']).items()
}
if alias_count:
node_info.update_capabilities(**alias_count)
LOG.info(_LI('Found the following PCI devices: %s'), alias_count)
def init():
if utils.get_auth_strategy() != 'noauth':
utils.add_auth_middleware(app)
else:
LOG.warning(_LW('Starting unauthenticated, please check'
' configuration'))
if CONF.processing.store_data == 'none':
LOG.warning(_LW('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not the '
'desired behavior'))
elif CONF.processing.store_data == 'swift':
LOG.info(_LI('Introspection data will be stored in Swift in the '
'container %s'), CONF.swift.container)
db.init()
try:
hooks = [ext.name for ext in plugins_base.processing_hooks_manager()]
except KeyError as exc:
# stevedore raises KeyError on missing hook
LOG.critical(_LC('Hook %s failed to load or was not found'), str(exc))
sys.exit(1)
LOG.info(_LI('Enabled processing hooks: %s'), hooks)
if CONF.firewall.manage_firewall:
firewall.init()
period = CONF.firewall.firewall_update_period
utils.spawn_n(periodic_update, period)
if CONF.timeout > 0:
period = CONF.clean_up_period
utils.spawn_n(periodic_clean_up, period)
else:
LOG.warning(_LW('Timeout is disabled in configuration'))
def _process_node(node, introspection_data, node_info):
# NOTE(dtantsur): repeat the check in case something changed
utils.check_provision_state(node)
node_info.create_ports(introspection_data.get('macs') or ())
_run_post_hooks(node_info, introspection_data)
if CONF.processing.store_data == 'swift':
swift_object_name = swift.store_introspection_data(
introspection_data, node_info.uuid)
LOG.info(
_LI('Introspection data for node %(node)s was stored in '
'Swift in object %(obj)s'), {
'node': node_info.uuid,
'obj': swift_object_name
})
if CONF.processing.store_data_location:
node_info.patch([{
'op': 'add',
'path': '/extra/%s' % CONF.processing.store_data_location,
'value': swift_object_name
}])
else:
LOG.debug(
'Swift support is disabled, introspection data for node %s '
'won\'t be stored', node_info.uuid)
ironic = utils.get_client()
firewall.update_filters(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.uuid}
if node_info.options.get('new_ipmi_credentials'):
new_username, new_password = (
node_info.options.get('new_ipmi_credentials'))
utils.spawn_n(_finish_set_ipmi_credentials, ironic, node, node_info,
introspection_data, new_username, new_password)
resp['ipmi_setup_credentials'] = True
resp['ipmi_username'] = new_username
resp['ipmi_password'] = new_password
else:
utils.spawn_n(_finish, ironic, node_info)
return resp
def _finish(ironic, node_info):
LOG.debug('Forcing power off of node %s', node_info.uuid)
try:
ironic.node.set_power_state(node_info.uuid, 'off')
except Exception as exc:
msg = (_('Failed to power off node %(node)s, check it\'s power '
'management configuration: %(exc)s') % {
'node': node_info.uuid,
'exc': exc
})
node_info.finished(error=msg)
raise utils.Error(msg)
node_info.finished()
LOG.info(_LI('Introspection finished successfully for node %s'),
node_info.uuid)
def _process_node(node, introspection_data, node_info):
# NOTE(dtantsur): repeat the check in case something changed
ir_utils.check_provision_state(node)
node_info.create_ports(introspection_data.get('macs') or ())
_run_post_hooks(node_info, introspection_data)
if CONF.processing.store_data == 'swift':
stored_data = {k: v for k, v in introspection_data.items()
if k not in _STORAGE_EXCLUDED_KEYS}
swift_object_name = swift.store_introspection_data(stored_data,
node_info.uuid)
LOG.info(_LI('Introspection data was stored in Swift in object %s'),
swift_object_name,
node_info=node_info, data=introspection_data)
if CONF.processing.store_data_location:
node_info.patch([{'op': 'add', 'path': '/extra/%s' %
CONF.processing.store_data_location,
'value': swift_object_name}])
else:
LOG.debug('Swift support is disabled, introspection data '
'won\'t be stored',
node_info=node_info, data=introspection_data)
ironic = ir_utils.get_client()
firewall.update_filters(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.uuid}
if node_info.options.get('new_ipmi_credentials'):
new_username, new_password = (
node_info.options.get('new_ipmi_credentials'))
utils.executor().submit(_finish_set_ipmi_credentials,
ironic, node, node_info, introspection_data,
new_username, new_password)
resp['ipmi_setup_credentials'] = True
resp['ipmi_username'] = new_username
resp['ipmi_password'] = new_password
else:
utils.executor().submit(_finish, ironic, node_info, introspection_data)
return resp
def _store_data(node_info, data, suffix=None):
if CONF.processing.store_data != 'swift':
LOG.debug("Swift support is disabled, introspection data "
"won't be stored", node_info=node_info)
return
swift_object_name = swift.store_introspection_data(
_filter_data_excluded_keys(data),
node_info.uuid,
suffix=suffix
)
LOG.info(_LI('Introspection data was stored in Swift in object '
'%s'), swift_object_name, node_info=node_info)
if CONF.processing.store_data_location:
node_info.patch([{'op': 'add', 'path': '/extra/%s' %
CONF.processing.store_data_location,
'value': swift_object_name}])
def _store_data(node_info, data, suffix=None):
if CONF.processing.store_data != 'swift':
LOG.debug("Swift support is disabled, introspection data "
"won't be stored", node_info=node_info)
return
swift_object_name = swift.store_introspection_data(
_filter_data_excluded_keys(data),
node_info.uuid,
suffix=suffix
)
LOG.info(_LI('Introspection data was stored in Swift in object '
'%s'), swift_object_name, node_info=node_info)
if CONF.processing.store_data_location:
node_info.patch([{'op': 'add', 'path': '/extra/%s' %
CONF.processing.store_data_location,
'value': swift_object_name}])
def _detect_cpu_flags(self, inventory, node_info, data=None):
flags = inventory['cpu'].get('flags')
if not flags:
LOG.warning(_LW('No CPU flags available, please update your '
'introspection ramdisk'),
data=data, node_info=node_info)
return {}
flags = set(flags)
caps = {}
for flag, name in CONF.capabilities.cpu_flags.items():
if flag in flags:
caps[name] = 'true'
LOG.info(_LI('CPU capabilities: %s'), list(caps),
data=data, node_info=node_info)
return caps
def before_update(self, introspection_data, node_info, **kwargs):
"""Update node with scheduler properties."""
inventory = utils.get_inventory(introspection_data,
node_info=node_info)
errors = []
root_disk = introspection_data.get('root_disk')
if root_disk:
introspection_data['local_gb'] = root_disk['size'] // units.Gi
if CONF.processing.disk_partitioning_spacing:
introspection_data['local_gb'] -= 1
else:
introspection_data['local_gb'] = 0
try:
introspection_data['cpus'] = int(inventory['cpu']['count'])
introspection_data['cpu_arch'] = six.text_type(
inventory['cpu']['architecture'])
except (KeyError, ValueError, TypeError):
errors.append(_('malformed or missing CPU information: %s') %
inventory.get('cpu'))
try:
introspection_data['memory_mb'] = int(
inventory['memory']['physical_mb'])
except (KeyError, ValueError, TypeError):
errors.append(_('malformed or missing memory information: %s; '
'introspection requires physical memory size '
'from dmidecode') % inventory.get('memory'))
if errors:
raise utils.Error(_('The following problems encountered: %s') %
'; '.join(errors),
node_info=node_info, data=introspection_data)
LOG.info(_LI('Discovered data: CPUs: %(cpus)s %(cpu_arch)s, '
'memory %(memory_mb)s MiB, disk %(local_gb)s GiB'),
{key: introspection_data.get(key) for key in self.KEYS},
node_info=node_info, data=introspection_data)
overwrite = CONF.processing.overwrite_existing
properties = {key: str(introspection_data[key])
for key in self.KEYS if overwrite or
not node_info.node().properties.get(key)}
node_info.update_properties(**properties)
def _reapply(node_info):
# runs in background
try:
introspection_data = _get_unprocessed_data(node_info.uuid)
except Exception as exc:
LOG.exception(_LE('Encountered exception while fetching '
'stored introspection data'),
node_info=node_info)
msg = (_('Unexpected exception %(exc_class)s while fetching '
'unprocessed introspection data from Swift: %(error)s') % {
'exc_class': exc.__class__.__name__,
'error': exc
})
node_info.finished(error=msg)
return
failures = []
_run_pre_hooks(introspection_data, failures)
if failures:
LOG.error(_LE('Pre-processing failures detected reapplying '
'introspection on stored data:\n%s'),
'\n'.join(failures),
node_info=node_info)
node_info.finished(error='\n'.join(failures))
return
try:
ironic = ir_utils.get_client()
node_info.create_ports(introspection_data.get('macs') or ())
_run_post_hooks(node_info, introspection_data)
_store_data(node_info, introspection_data)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
_finish(ironic, node_info, introspection_data, power_off=False)
except Exception as exc:
LOG.exception(_LE('Encountered exception reapplying '
'introspection on stored data'),
node_info=node_info,
data=introspection_data)
node_info.finished(error=str(exc))
else:
LOG.info(_LI('Successfully reapplied introspection on stored '
'data'),
node_info=node_info,
data=introspection_data)
def _store_logs(self, logs, introspection_data):
if not CONF.processing.ramdisk_logs_dir:
LOG.warning(
_LW('Failed to store logs received from the ramdisk '
'because ramdisk_logs_dir configuration option '
'is not set'))
return
if not os.path.exists(CONF.processing.ramdisk_logs_dir):
os.makedirs(CONF.processing.ramdisk_logs_dir)
time_fmt = datetime.datetime.utcnow().strftime(self.DATETIME_FORMAT)
bmc_address = introspection_data.get('ipmi_address', 'unknown')
file_name = 'bmc_%s_%s' % (bmc_address, time_fmt)
with open(os.path.join(CONF.processing.ramdisk_logs_dir, file_name),
'wb') as fp:
fp.write(base64.b64decode(logs))
LOG.info(_LI('Ramdisk logs stored in file %s'), file_name)
def before_update(self, introspection_data, node_info, **kwargs):
current_devices = self._get_serials(introspection_data)
if not current_devices:
LOG.warning(_LW('No block device was received from ramdisk'),
node_info=node_info, data=introspection_data)
return
node = node_info.node()
if 'root_device' in node.properties:
LOG.info(_LI('Root device is already known for the node'),
node_info=node_info, data=introspection_data)
return
if 'block_devices' in node.extra:
# Compare previously discovered devices with the current ones
previous_devices = node.extra['block_devices']['serials']
new_devices = [device for device in current_devices
if device not in previous_devices]
if len(new_devices) > 1:
LOG.warning(_LW('Root device cannot be identified because '
'multiple new devices were found'),
node_info=node_info, data=introspection_data)
return
elif len(new_devices) == 0:
LOG.warning(_LW('No new devices were found'),
node_info=node_info, data=introspection_data)
return
node_info.patch([
{'op': 'remove',
'path': '/extra/block_devices'},
{'op': 'add',
'path': '/properties/root_device',
'value': {'serial': new_devices[0]}}
])
else:
# No previously discovered devices - save the inspector block
# devices in node.extra
node_info.patch([{'op': 'add',
'path': '/extra/block_devices',
'value': {'serials': current_devices}}])
def _process_node(node, introspection_data, node_info):
# NOTE(dtantsur): repeat the check in case something changed
utils.check_provision_state(node)
node_info.create_ports(introspection_data.get('macs') or ())
_run_post_hooks(node_info, introspection_data)
if CONF.processing.store_data == 'swift':
swift_object_name = swift.store_introspection_data(introspection_data,
node_info.uuid)
LOG.info(_LI('Introspection data for node %(node)s was stored in '
'Swift in object %(obj)s'),
{'node': node_info.uuid, 'obj': swift_object_name})
if CONF.processing.store_data_location:
node_info.patch([{'op': 'add', 'path': '/extra/%s' %
CONF.processing.store_data_location,
'value': swift_object_name}])
else:
LOG.debug('Swift support is disabled, introspection data for node %s '
'won\'t be stored', node_info.uuid)
ironic = utils.get_client()
firewall.update_filters(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.uuid}
if node_info.options.get('new_ipmi_credentials'):
new_username, new_password = (
node_info.options.get('new_ipmi_credentials'))
utils.spawn_n(_finish_set_ipmi_credentials,
ironic, node, node_info, introspection_data,
new_username, new_password)
resp['ipmi_setup_credentials'] = True
resp['ipmi_username'] = new_username
resp['ipmi_password'] = new_password
else:
utils.spawn_n(_finish, ironic, node_info)
return resp
def get_inventory(data, node_info=None):
"""Get and validate the hardware inventory from introspection data."""
inventory = data.get('inventory')
# TODO(dtantsur): validate inventory using JSON schema
if not inventory:
raise Error(_('Hardware inventory is empty or missing'),
data=data, node_info=node_info)
for key in _INVENTORY_MANDATORY_KEYS:
if not inventory.get(key):
raise Error(_('Invalid hardware inventory: %s key is missing '
'or empty') % key, data=data, node_info=node_info)
if not inventory.get('disks'):
LOG.info(_LI('No disks were detected in the inventory, assuming this '
'is a disk-less node'), data=data, node_info=node_info)
# Make sure the code iterating over it does not fail with a TypeError
inventory['disks'] = []
return inventory
def create_node(driver, ironic=None, **attributes):
"""Create ironic node and cache it.
* Create new node in ironic.
* Cache it in inspector.
:param driver: driver for Ironic node.
:param ironic: ronic client instance.
:param attributes: dict, additional keyword arguments to pass
to the ironic client on node creation.
:return: NodeInfo, or None in case error happened.
"""
if ironic is None:
ironic = ir_utils.get_client()
try:
node = ironic.node.create(driver=driver, **attributes)
except exceptions.InvalidAttribute as e:
LOG.error(_LE('Failed to create new node: %s'), e)
else:
LOG.info(_LI('Node %s was created successfully'), node.uuid)
return add_node(node.uuid, ironic=ironic)
def before_update(self, introspection_data, node_info, **kwargs):
"""Drop ports that are not present in the data."""
if CONF.processing.keep_ports == 'present':
expected_macs = {
iface['mac']
for iface in introspection_data['all_interfaces'].values()
}
elif CONF.processing.keep_ports == 'added':
expected_macs = set(introspection_data['macs'])
else:
return
# list is required as we modify underlying dict
for port in list(node_info.ports().values()):
if port.address not in expected_macs:
LOG.info(_LI("Deleting port %(port)s as its MAC %(mac)s is "
"not in expected MAC list %(expected)s"),
{'port': port.uuid,
'mac': port.address,
'expected': list(sorted(expected_macs))},
node_info=node_info, data=introspection_data)
node_info.delete_port(port)