def _abort(node_info, ironic): # runs in background if node_info.finished_at is not None: # introspection already finished; nothing to do LOG.info(_LI('Cannot abort introspection as it is already ' 'finished'), node_info=node_info) node_info.release_lock() return # block this node from PXE Booting the introspection image try: firewall.update_filters(ironic) except Exception as exc: # Note(mkovacik): this will be retried in firewall update # periodic task; we continue aborting LOG.warning(_LW('Failed to update firewall filters: %s'), exc, node_info=node_info) # finish the introspection LOG.debug('Forcing power-off', node_info=node_info) try: ironic.node.set_power_state(node_info.uuid, 'off') except Exception as exc: LOG.warning(_LW('Failed to power off node: %s'), exc, node_info=node_info) node_info.finished(error=_('Canceled by operator')) LOG.info(_LI('Introspection aborted'), node_info=node_info)
def _finish_common(node_info, ironic, introspection_data, power_off=True): if power_off: LOG.debug('Forcing power off of node %s', node_info.uuid) try: ironic.node.set_power_state(node_info.uuid, 'off') except Exception as exc: if node_info.node().provision_state == 'enroll': LOG.info(_LI("Failed to power off the node in" "'enroll' state, ignoring; error was " "%s"), exc, node_info=node_info, data=introspection_data) else: msg = (_('Failed to power off node %(node)s, check ' 'its power management configuration: ' '%(exc)s') % {'node': node_info.uuid, 'exc': exc}) node_info.finished(error=msg) raise utils.Error(msg, node_info=node_info, data=introspection_data) LOG.info(_LI('Node powered-off'), node_info=node_info, data=introspection_data) node_info.finished() LOG.info(_LI('Introspection finished successfully'), node_info=node_info, data=introspection_data)
def _finish(ironic, node_info, introspection_data, power_off=True): if power_off: LOG.debug('Forcing power off of node %s', node_info.uuid) try: ironic.node.set_power_state(node_info.uuid, 'off') except Exception as exc: if node_info.node().provision_state == 'enroll': LOG.info(_LI("Failed to power off the node in" "'enroll' state, ignoring; error was " "%s") % exc, node_info=node_info, data=introspection_data) else: msg = (_('Failed to power off node %(node)s, check ' 'its power management configuration: ' '%(exc)s') % {'node': node_info.uuid, 'exc': exc}) node_info.finished(error=msg) raise utils.Error(msg, node_info=node_info, data=introspection_data) LOG.info(_LI('Node powered-off'), node_info=node_info, data=introspection_data) node_info.finished() LOG.info(_LI('Introspection finished successfully'), node_info=node_info, data=introspection_data)
def _background_introspect_locked(ironic, node_info): # TODO(dtantsur): pagination macs = list(node_info.ports()) if macs: node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs) LOG.info( _LI('Whitelisting MAC\'s %(macs)s for node %(node)s on the' ' firewall') % { 'macs': macs, 'node': node_info.uuid }) firewall.update_filters(ironic) attrs = node_info.attributes if CONF.processing.node_not_found_hook is None and not attrs: raise utils.Error( _('No lookup attributes were found for node %s, inspector won\'t ' 'be able to find it after introspection. Consider creating ' 'ironic ports or providing an IPMI address.') % node_info.uuid) LOG.info( _LI('The following attributes will be used for looking up ' 'node %(uuid)s: %(attrs)s'), { 'attrs': attrs, 'uuid': node_info.uuid }) if not node_info.options.get('new_ipmi_credentials'): try: ironic.node.set_boot_device(node_info.uuid, 'pxe', persistent=False) except Exception as exc: LOG.warning( _LW('Failed to set boot device to PXE for' ' node %(node)s: %(exc)s') % { 'node': node_info.uuid, 'exc': exc }) try: ironic.node.set_power_state(node_info.uuid, 'reboot') except Exception as exc: raise utils.Error( _('Failed to power on node %(node)s,' ' check it\'s power ' 'management configuration:\n%(exc)s') % { 'node': node_info.uuid, 'exc': exc }) LOG.info(_LI('Introspection started successfully for node %s'), node_info.uuid) else: LOG.info( _LI('Introspection environment is ready for node %(node)s, ' 'manual power on is required within %(timeout)d seconds') % { 'node': node_info.uuid, 'timeout': CONF.timeout })
def _background_introspect(ironic, node_info): global _LAST_INTROSPECTION_TIME # TODO(dtantsur): pagination macs = list(node_info.ports()) if macs: node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs) LOG.info(_LI('Whitelisting MAC\'s %(macs)s for node %(node)s on the' ' firewall') % {'macs': macs, 'node': node_info.uuid}) firewall.update_filters(ironic) attrs = node_info.attributes if CONF.processing.node_not_found_hook is None and not attrs: raise utils.Error( _('No lookup attributes were found for node %s, inspector won\'t ' 'be able to find it after introspection. Consider creating ' 'ironic ports or providing an IPMI address.') % node_info.uuid) LOG.info(_LI('The following attributes will be used for looking up ' 'node %(uuid)s: %(attrs)s'), {'attrs': attrs, 'uuid': node_info.uuid}) if not node_info.options.get('new_ipmi_credentials'): try: ironic.node.set_boot_device(node_info.uuid, 'pxe', persistent=False) except Exception as exc: LOG.warning(_LW('Failed to set boot device to PXE for' ' node %(node)s: %(exc)s') % {'node': node_info.uuid, 'exc': exc}) if re.match(CONF.introspection_delay_drivers, node_info.node().driver): LOG.debug('Attempting to acquire lock on last introspection time') with _LAST_INTROSPECTION_LOCK: delay = (_LAST_INTROSPECTION_TIME - time.time() + CONF.introspection_delay) if delay > 0: LOG.debug('Waiting %d seconds before sending the next ' 'node on introspection', delay) time.sleep(delay) _LAST_INTROSPECTION_TIME = time.time() try: ironic.node.set_power_state(node_info.uuid, 'reboot') except Exception as exc: raise utils.Error(_('Failed to power on node %(node)s,' ' check it\'s power ' 'management configuration:\n%(exc)s') % {'node': node_info.uuid, 'exc': exc}) LOG.info(_LI('Introspection started successfully for node %s'), node_info.uuid) else: LOG.info(_LI('Introspection environment is ready for node %(node)s, ' 'manual power on is required within %(timeout)d seconds') % {'node': node_info.uuid, 'timeout': CONF.timeout})
def check_conditions(self, node_info, data): """Check if conditions are true for a given node. :param node_info: a NodeInfo object :param data: introspection data :returns: True if conditions match, otherwise False """ LOG.debug('Checking rule "%s"', self.description, node_info=node_info, data=data) ext_mgr = plugins_base.rule_conditions_manager() for cond in self._conditions: scheme, path = _parse_path(cond.field) if scheme == 'node': source_data = node_info.node().to_dict() elif scheme == 'data': source_data = data field_values = jsonpath.parse(path).find(source_data) field_values = [x.value for x in field_values] cond_ext = ext_mgr[cond.op].obj if not field_values: if cond_ext.ALLOW_NONE: LOG.debug('Field with JSON path %s was not found in data', cond.field, node_info=node_info, data=data) field_values = [None] else: LOG.info(_LI('Field with JSON path %(path)s was not found ' 'in data, rule "%(rule)s" will not ' 'be applied'), {'path': cond.field, 'rule': self.description}, node_info=node_info, data=data) return False for value in field_values: result = cond_ext.check(node_info, value, cond.params) if cond.invert: result = not result if (cond.multiple == 'first' or (cond.multiple == 'all' and not result) or (cond.multiple == 'any' and result)): break if not result: LOG.info(_LI('Rule "%(rule)s" will not be applied: condition ' '%(field)s %(op)s %(params)s failed'), {'rule': self.description, 'field': cond.field, 'op': cond.op, 'params': cond.params}, node_info=node_info, data=data) return False LOG.info(_LI('Rule "%s" will be applied'), self.description, node_info=node_info, data=data) return True
def before_processing(self, introspection_data, **kwargs): """Validate information about network interfaces.""" bmc_address = introspection_data.get('ipmi_address') if not introspection_data.get('interfaces'): raise utils.Error(_('No interfaces supplied by the ramdisk')) valid_interfaces = { n: iface for n, iface in introspection_data['interfaces'].items() if utils.is_valid_mac(iface.get('mac')) } pxe_mac = introspection_data.get('boot_interface') if CONF.processing.add_ports == 'pxe' and not pxe_mac: LOG.warning(_LW('No boot interface provided in the introspection ' 'data, will add all ports with IP addresses')) if CONF.processing.add_ports == 'pxe' and pxe_mac: LOG.info(_LI('PXE boot interface was %s'), pxe_mac) if '-' in pxe_mac: # pxelinux format: 01-aa-bb-cc-dd-ee-ff pxe_mac = pxe_mac.split('-', 1)[1] pxe_mac = pxe_mac.replace('-', ':').lower() valid_interfaces = { n: iface for n, iface in valid_interfaces.items() if iface['mac'].lower() == pxe_mac } elif CONF.processing.add_ports != 'all': valid_interfaces = { n: iface for n, iface in valid_interfaces.items() if iface.get('ip') } if not valid_interfaces: raise utils.Error(_('No valid interfaces found for node with ' 'BMC %(ipmi_address)s, got %(interfaces)s') % {'ipmi_address': bmc_address, 'interfaces': introspection_data['interfaces']}) elif valid_interfaces != introspection_data['interfaces']: invalid = {n: iface for n, iface in introspection_data['interfaces'].items() if n not in valid_interfaces} LOG.warning(_LW( 'The following interfaces were invalid or not eligible in ' 'introspection data for node with BMC %(ipmi_address)s and ' 'were excluded: %(invalid)s'), {'invalid': invalid, 'ipmi_address': bmc_address}) LOG.info(_LI('Eligible interfaces are %s'), valid_interfaces) introspection_data['all_interfaces'] = introspection_data['interfaces'] introspection_data['interfaces'] = valid_interfaces valid_macs = [iface['mac'] for iface in valid_interfaces.values()] introspection_data['macs'] = valid_macs
def check_conditions(self, node_info, data): """Check if conditions are true for a given node. :param node_info: a NodeInfo object :param data: introspection data :returns: True if conditions match, otherwise False """ LOG.debug('Checking rule "%(descr)s" on node %(uuid)s', {'descr': self.description, 'uuid': node_info.uuid}) ext_mgr = plugins_base.rule_conditions_manager() for cond in self._conditions: field_values = jsonpath.parse(cond.field).find(data) field_values = [x.value for x in field_values] cond_ext = ext_mgr[cond.op].obj if not field_values: if cond_ext.ALLOW_NONE: LOG.debug('Field with JSON path %(path)s was not found in ' 'data for node %(uuid)s', {'path': cond.field, 'uuid': node_info.uuid}) field_values = [None] else: LOG.info(_LI('Field with JSON path %(path)s was not found ' 'in data for node %(uuid)s, rule "%(rule)s" ' 'will not be applied'), {'path': cond.field, 'uuid': node_info.uuid, 'rule': self.description}) return False for value in field_values: result = cond_ext.check(node_info, value, cond.params) if (cond.multiple == 'first' or (cond.multiple == 'all' and not result) or (cond.multiple == 'any' and result)): break if not result: LOG.info(_LI('Rule "%(rule)s" will not be applied to node ' '%(uuid)s: condition %(field)s %(op)s %(params)s ' 'failed'), {'rule': self.description, 'uuid': node_info.uuid, 'field': cond.field, 'op': cond.op, 'params': cond.params}) return False LOG.info(_LI('Rule "%(rule)s" will be applied to node %(uuid)s'), {'rule': self.description, 'uuid': node_info.uuid}) return True
def _finish_set_ipmi_credentials(ironic, node, node_info, introspection_data, new_username, new_password): patch = [{'op': 'add', 'path': '/driver_info/ipmi_username', 'value': new_username}, {'op': 'add', 'path': '/driver_info/ipmi_password', 'value': new_password}] if (not ir_utils.get_ipmi_address(node) and introspection_data.get('ipmi_address')): patch.append({'op': 'add', 'path': '/driver_info/ipmi_address', 'value': introspection_data['ipmi_address']}) node_info.patch(patch) for attempt in range(_CREDENTIALS_WAIT_RETRIES): try: # We use this call because it requires valid credentials. # We don't care about boot device, obviously. ironic.node.get_boot_device(node_info.uuid) except Exception as exc: LOG.info(_LI('Waiting for credentials update, attempt %(attempt)d ' 'current error is %(exc)s') % {'attempt': attempt, 'exc': exc}, node_info=node_info, data=introspection_data) eventlet.greenthread.sleep(_CREDENTIALS_WAIT_PERIOD) else: _finish(ironic, node_info, introspection_data) return msg = (_('Failed to validate updated IPMI credentials for node ' '%s, node might require maintenance') % node_info.uuid) node_info.finished(error=msg) raise utils.Error(msg, node_info=node_info, data=introspection_data)
def before_processing(self, introspection_data, **kwargs): """Validate information about network interfaces.""" bmc_address = utils.get_ipmi_address_from_data(introspection_data) if bmc_address: introspection_data['ipmi_address'] = bmc_address else: LOG.debug( 'No BMC address provided in introspection data, ' 'assuming virtual environment', data=introspection_data) all_interfaces = self._get_interfaces(introspection_data) interfaces = self._validate_interfaces(all_interfaces, introspection_data) LOG.info(_LI('Using network interface(s): %s'), ', '.join('%s %s' % (name, items) for (name, items) in interfaces.items()), data=introspection_data) introspection_data['all_interfaces'] = all_interfaces introspection_data['interfaces'] = interfaces valid_macs = [iface['mac'] for iface in interfaces.values()] introspection_data['macs'] = valid_macs
def before_update(self, introspection_data, node_info, **kwargs): """Drop ports that are not present in the data.""" if CONF.processing.keep_ports == 'present': expected_macs = { iface['mac'] for iface in introspection_data['all_interfaces'].values() } elif CONF.processing.keep_ports == 'added': expected_macs = set(introspection_data['macs']) else: return # list is required as we modify underlying dict for port in list(node_info.ports().values()): if port.address not in expected_macs: LOG.info( _LI("Deleting port %(port)s as its MAC %(mac)s is " "not in expected MAC list %(expected)s for node " "%(node)s"), { 'port': port.uuid, 'mac': port.address, 'expected': list(sorted(expected_macs)), 'node': node_info.uuid }) node_info.delete_port(port)
def before_update(self, introspection_data, node_info, **kwargs): """Update node with scheduler properties.""" root_disk = introspection_data.get('root_disk') if root_disk: introspection_data['local_gb'] = root_disk['size'] // units.Gi if CONF.processing.disk_partitioning_spacing: introspection_data['local_gb'] -= 1 missing = [key for key in self.KEYS if not introspection_data.get(key)] if missing: raise utils.Error( _('The following required parameters are missing: %s') % missing) LOG.info( _LI('Discovered data: CPUs: %(cpus)s %(cpu_arch)s, ' 'memory %(memory_mb)s MiB, disk %(local_gb)s GiB'), {key: introspection_data.get(key) for key in self.KEYS}) overwrite = CONF.processing.overwrite_existing properties = { key: str(introspection_data[key]) for key in self.KEYS if overwrite or not node_info.node().properties.get(key) } node_info.update_properties(**properties)
def before_processing(self, introspection_data, **kwargs): """Adds fake local_gb value if it's missing from introspection_data.""" if not introspection_data.get('local_gb'): LOG.info(_LI('No volume is found on the node. Adding a fake ' 'value for "local_gb"'), data=introspection_data) introspection_data['local_gb'] = 1
def apply(node_info, data): """Apply rules to a node.""" rules = get_all() if not rules: LOG.debug('No custom introspection rules to apply to node %s', node_info.uuid) return LOG.debug('Applying custom introspection rules to node %s', node_info.uuid) to_rollback = [] to_apply = [] for rule in rules: if rule.check_conditions(node_info, data): to_apply.append(rule) else: to_rollback.append(rule) if to_rollback: LOG.debug('Running rollback actions on node %s', node_info.uuid) for rule in to_rollback: rule.apply_actions(node_info, rollback=True) else: LOG.debug('No rollback actions to apply on node %s', node_info.uuid) if to_apply: LOG.debug('Running actions on node %s', node_info.uuid) for rule in to_apply: rule.apply_actions(node_info, rollback=False) else: LOG.debug('No actions to apply on node %s', node_info.uuid) LOG.info(_LI('Successfully applied custom introspection rules to node %s'), node_info.uuid)
def before_update(self, introspection_data, node_info, **kwargs): """Update node with scheduler properties.""" inventory = introspection_data.get('inventory') errors = [] root_disk = introspection_data.get('root_disk') if root_disk: introspection_data['local_gb'] = root_disk['size'] // units.Gi if CONF.processing.disk_partitioning_spacing: introspection_data['local_gb'] -= 1 elif inventory: errors.append(_('root disk is not supplied by the ramdisk and ' 'root_disk_selection hook is not enabled')) if inventory: try: introspection_data['cpus'] = int(inventory['cpu']['count']) introspection_data['cpu_arch'] = six.text_type( inventory['cpu']['architecture']) except (KeyError, ValueError, TypeError): errors.append(_('malformed or missing CPU information: %s') % inventory.get('cpu')) try: introspection_data['memory_mb'] = int( inventory['memory']['physical_mb']) except (KeyError, ValueError, TypeError): errors.append(_('malformed or missing memory information: %s; ' 'introspection requires physical memory size ' 'from dmidecode') % inventory.get('memory')) else: LOG.warning(_LW('No inventory provided: using old bash ramdisk ' 'is deprecated, please switch to ' 'ironic-python-agent'), node_info=node_info, data=introspection_data) missing = [key for key in self.KEYS if not introspection_data.get(key)] if missing: raise utils.Error( _('The following required parameters are missing: %s') % missing, node_info=node_info, data=introspection_data) if errors: raise utils.Error(_('The following problems encountered: %s') % '; '.join(errors), node_info=node_info, data=introspection_data) LOG.info(_LI('Discovered data: CPUs: %(cpus)s %(cpu_arch)s, ' 'memory %(memory_mb)s MiB, disk %(local_gb)s GiB'), {key: introspection_data.get(key) for key in self.KEYS}, node_info=node_info, data=introspection_data) overwrite = CONF.processing.overwrite_existing properties = {key: str(introspection_data[key]) for key in self.KEYS if overwrite or not node_info.node().properties.get(key)} node_info.update_properties(**properties)
def _finish_set_ipmi_credentials(node_info, ironic, node, introspection_data, new_username, new_password): patch = [{'op': 'add', 'path': '/driver_info/ipmi_username', 'value': new_username}, {'op': 'add', 'path': '/driver_info/ipmi_password', 'value': new_password}] new_ipmi_address = utils.get_ipmi_address_from_data(introspection_data) if not ir_utils.get_ipmi_address(node) and new_ipmi_address: patch.append({'op': 'add', 'path': '/driver_info/ipmi_address', 'value': new_ipmi_address}) node_info.patch(patch) for attempt in range(_CREDENTIALS_WAIT_RETRIES): try: # We use this call because it requires valid credentials. # We don't care about boot device, obviously. ironic.node.get_boot_device(node_info.uuid) except Exception as exc: LOG.info(_LI('Waiting for credentials update, attempt %(attempt)d ' 'current error is %(exc)s'), {'attempt': attempt, 'exc': exc}, node_info=node_info, data=introspection_data) eventlet.greenthread.sleep(_CREDENTIALS_WAIT_PERIOD) else: _finish_common(node_info, ironic, introspection_data) return msg = (_('Failed to validate updated IPMI credentials for node ' '%s, node might require maintenance') % node_info.uuid) node_info.finished(error=msg) raise utils.Error(msg, node_info=node_info, data=introspection_data)
def _reapply(node_info): # runs in background try: introspection_data = _get_unprocessed_data(node_info.uuid) except Exception as exc: LOG.exception(_LE('Encountered exception while fetching ' 'stored introspection data'), node_info=node_info) msg = (_('Unexpected exception %(exc_class)s while fetching ' 'unprocessed introspection data from Swift: %(error)s') % {'exc_class': exc.__class__.__name__, 'error': exc}) node_info.finished(error=msg) return try: ironic = ir_utils.get_client() except Exception as exc: msg = _('Encountered an exception while getting the Ironic client: ' '%s') % exc LOG.error(msg, node_info=node_info, data=introspection_data) node_info.fsm_event(istate.Events.error) node_info.finished(error=msg) return try: _reapply_with_data(node_info, introspection_data) except Exception as exc: node_info.finished(error=str(exc)) return _finish(node_info, ironic, introspection_data, power_off=False) LOG.info(_LI('Successfully reapplied introspection on stored ' 'data'), node_info=node_info, data=introspection_data)
def init(self): if utils.get_auth_strategy() != 'noauth': utils.add_auth_middleware(app) else: LOG.warning(_LW('Starting unauthenticated, please check' ' configuration')) if CONF.processing.store_data == 'none': LOG.warning(_LW('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior')) elif CONF.processing.store_data == 'swift': LOG.info(_LI('Introspection data will be stored in Swift in the ' 'container %s'), CONF.swift.container) utils.add_cors_middleware(app) db.init() try: hooks = [ext.name for ext in plugins_base.processing_hooks_manager()] except KeyError as exc: # callback function raises MissingHookError derived from KeyError # on missing hook LOG.critical(_LC('Hook(s) %s failed to load or was not found'), str(exc)) sys.exit(1) LOG.info(_LI('Enabled processing hooks: %s'), hooks) if CONF.firewall.manage_firewall: firewall.init() periodic_update_ = periodics.periodic( spacing=CONF.firewall.firewall_update_period, enabled=CONF.firewall.manage_firewall )(periodic_update) periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period )(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(periodic_update_, None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor())) utils.executor().submit(self._periodics_worker.start)
def delete_all(): """Delete all rules.""" with db.ensure_transaction() as session: db.model_query(db.RuleAction, session=session).delete() db.model_query(db.RuleCondition, session=session).delete() db.model_query(db.Rule, session=session).delete() LOG.info(_LI('All introspection rules were deleted'))
def _background_introspect_locked(ironic, node_info): # TODO(dtantsur): pagination macs = list(node_info.ports()) if macs: node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs) LOG.info(_LI('Whitelisting MAC\'s %s on the firewall'), macs, node_info=node_info) firewall.update_filters(ironic) attrs = node_info.attributes if CONF.processing.node_not_found_hook is None and not attrs: raise utils.Error(_( 'No lookup attributes were found, inspector won\'t ' 'be able to find it after introspection, consider creating ' 'ironic ports or providing an IPMI address'), node_info=node_info) LOG.info(_LI('The following attributes will be used for look up: %s'), attrs, node_info=node_info) if not node_info.options.get('new_ipmi_credentials'): try: ironic.node.set_boot_device(node_info.uuid, 'pxe', persistent=False) except Exception as exc: LOG.warning(_LW('Failed to set boot device to PXE: %s'), exc, node_info=node_info) try: ironic.node.set_power_state(node_info.uuid, 'reboot') except Exception as exc: raise utils.Error(_('Failed to power on the node, check it\'s ' 'power management configuration: %s'), exc, node_info=node_info) LOG.info(_LI('Introspection started successfully'), node_info=node_info) else: LOG.info(_LI('Introspection environment is ready, manual power on is ' 'required within %d seconds'), CONF.timeout, node_info=node_info)
def before_update(self, introspection_data, node_info, **kwargs): current_devices = self._get_serials(introspection_data) if not current_devices: LOG.warning(_LW('No block device was received from ramdisk'), node_info=node_info, data=introspection_data) return node = node_info.node() if 'root_device' in node.properties: LOG.info(_LI('Root device is already known for the node'), node_info=node_info, data=introspection_data) return if 'block_devices' in node.extra: # Compare previously discovered devices with the current ones previous_devices = node.extra['block_devices']['serials'] new_devices = [ device for device in current_devices if device not in previous_devices ] if len(new_devices) > 1: LOG.warning(_LW('Root device cannot be identified because ' 'multiple new devices were found'), node_info=node_info, data=introspection_data) return elif len(new_devices) == 0: LOG.warning(_LW('No new devices were found'), node_info=node_info, data=introspection_data) return node_info.patch([{ 'op': 'remove', 'path': '/extra/block_devices' }, { 'op': 'add', 'path': '/properties/root_device', 'value': { 'serial': new_devices[0] } }]) else: # No previously discovered devices - save the inspector block # devices in node.extra node_info.patch([{ 'op': 'add', 'path': '/extra/block_devices', 'value': { 'serials': current_devices } }])
def process(introspection_data): """Process data from the ramdisk. This function heavily relies on the hooks to do the actual data processing. """ unprocessed_data = copy.deepcopy(introspection_data) failures = [] _run_pre_hooks(introspection_data, failures) node_info = _find_node_info(introspection_data, failures) if node_info: # Locking is already done in find_node() but may be not done in a # node_not_found hook node_info.acquire_lock() if failures or node_info is None: msg = _('The following failures happened during running ' 'pre-processing hooks:\n%s') % '\n'.join(failures) if node_info is not None: node_info.finished(error='\n'.join(failures)) raise utils.Error(msg, node_info=node_info, data=introspection_data) LOG.info(_LI('Matching node is %s'), node_info.uuid, node_info=node_info, data=introspection_data) if node_info.finished_at is not None: # race condition or introspection canceled raise utils.Error(_('Node processing already finished with ' 'error: %s') % node_info.error, node_info=node_info, code=400) # Note(mkovacik): store data now when we're sure that a background # thread won't race with other process() or introspect.abort() # call utils.executor().submit(_store_unprocessed_data, node_info, unprocessed_data) try: node = node_info.node() except exceptions.NotFound: msg = _('Node was found in cache, but is not found in Ironic') node_info.finished(error=msg) raise utils.Error(msg, code=404, node_info=node_info, data=introspection_data) try: return _process_node(node, introspection_data, node_info) except utils.Error as exc: node_info.finished(error=str(exc)) raise except Exception as exc: LOG.exception(_LE('Unexpected exception during processing')) msg = _('Unexpected exception %(exc_class)s during processing: ' '%(error)s') % {'exc_class': exc.__class__.__name__, 'error': exc} node_info.finished(error=msg) raise utils.Error(msg, node_info=node_info, data=introspection_data)
def _detect_boot_mode(self, inventory, node_info, data=None): boot_mode = inventory.get('boot', {}).get('current_boot_mode') if boot_mode is not None: LOG.info(_LI('Boot mode was %s'), boot_mode, data=data, node_info=node_info) return {'boot_mode': boot_mode} else: LOG.warning(_LW('No boot mode information available'), data=data, node_info=node_info) return {}
def init(self): if utils.get_auth_strategy() != 'noauth': utils.add_auth_middleware(app) else: LOG.warning( _LW('Starting unauthenticated, please check' ' configuration')) if CONF.processing.store_data == 'none': LOG.warning( _LW('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior')) elif CONF.processing.store_data == 'swift': LOG.info( _LI('Introspection data will be stored in Swift in the ' 'container %s'), CONF.swift.container) utils.add_cors_middleware(app) db.init() try: hooks = [ ext.name for ext in plugins_base.processing_hooks_manager() ] except KeyError as exc: # callback function raises MissingHookError derived from KeyError # on missing hook LOG.critical(_LC('Hook(s) %s failed to load or was not found'), str(exc)) sys.exit(1) LOG.info(_LI('Enabled processing hooks: %s'), hooks) if CONF.firewall.manage_firewall: firewall.init() self._periodics_worker = periodics.PeriodicWorker( callables=[(periodic_update, None, None), (periodic_clean_up, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor())) utils.executor().submit(self._periodics_worker.start)
def before_update(self, introspection_data, node_info, **kwargs): """Update node with scheduler properties.""" inventory = utils.get_inventory(introspection_data, node_info=node_info) errors = [] root_disk = introspection_data.get('root_disk') if root_disk: introspection_data['local_gb'] = root_disk['size'] // units.Gi if CONF.processing.disk_partitioning_spacing: introspection_data['local_gb'] -= 1 else: errors.append( _('root disk is not supplied by the ramdisk and ' 'root_disk_selection hook is not enabled')) try: introspection_data['cpus'] = int(inventory['cpu']['count']) introspection_data['cpu_arch'] = six.text_type( inventory['cpu']['architecture']) except (KeyError, ValueError, TypeError): errors.append( _('malformed or missing CPU information: %s') % inventory.get('cpu')) try: introspection_data['memory_mb'] = int( inventory['memory']['physical_mb']) except (KeyError, ValueError, TypeError): errors.append( _('malformed or missing memory information: %s; ' 'introspection requires physical memory size ' 'from dmidecode') % inventory.get('memory')) if errors: raise utils.Error(_('The following problems encountered: %s') % '; '.join(errors), node_info=node_info, data=introspection_data) LOG.info(_LI('Discovered data: CPUs: %(cpus)s %(cpu_arch)s, ' 'memory %(memory_mb)s MiB, disk %(local_gb)s GiB'), {key: introspection_data.get(key) for key in self.KEYS}, node_info=node_info, data=introspection_data) overwrite = CONF.processing.overwrite_existing properties = { key: str(introspection_data[key]) for key in self.KEYS if overwrite or not node_info.node().properties.get(key) } node_info.update_properties(**properties)
def _finish(ironic, node_info, introspection_data): LOG.debug('Forcing power off of node %s', node_info.uuid) try: ironic.node.set_power_state(node_info.uuid, 'off') except Exception as exc: if node_info.node().provision_state == 'enroll': LOG.info(_LI("Failed to power off the node in 'enroll' state, " "ignoring; error was %s") % exc, node_info=node_info, data=introspection_data) else: msg = (_('Failed to power off node %(node)s, check it\'s ' 'power management configuration: %(exc)s') % {'node': node_info.uuid, 'exc': exc}) node_info.finished(error=msg) raise utils.Error(msg, node_info=node_info, data=introspection_data) node_info.finished() LOG.info(_LI('Introspection finished successfully'), node_info=node_info, data=introspection_data)
def _fsm_ctx(self): fsm = self._get_fsm() try: yield fsm finally: if fsm.current_state != self.state: LOG.info(_LI('Updating node state: %(current)s --> %(new)s'), { 'current': self.state, 'new': fsm.current_state }, node_info=self) self._set_state(fsm.current_state)
def _background_introspect_locked(ironic, node_info): # TODO(dtantsur): pagination macs = list(node_info.ports()) if macs: node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs) LOG.info(_LI('Whitelisting MAC\'s %s on the firewall'), macs, node_info=node_info) firewall.update_filters(ironic) attrs = node_info.attributes if CONF.processing.node_not_found_hook is None and not attrs: raise utils.Error( _('No lookup attributes were found, inspector won\'t ' 'be able to find it after introspection, consider creating ' 'ironic ports or providing an IPMI address'), node_info=node_info) LOG.info(_LI('The following attributes will be used for look up: %s'), attrs, node_info=node_info) if not node_info.options.get('new_ipmi_credentials'): try: ironic.node.set_boot_device(node_info.uuid, 'pxe', persistent=False) except Exception as exc: LOG.warning(_LW('Failed to set boot device to PXE: %s'), exc, node_info=node_info) try: ironic.node.set_power_state(node_info.uuid, 'reboot') except Exception as exc: raise utils.Error(_('Failed to power on the node, check it\'s ' 'power management configuration: %s'), exc, node_info=node_info) LOG.info(_LI('Introspection started successfully'), node_info=node_info) else: LOG.info(_LI('Introspection environment is ready, manual power on is ' 'required within %d seconds'), CONF.timeout, node_info=node_info)
def init(): if utils.get_auth_strategy() != 'noauth': utils.add_auth_middleware(app) else: LOG.warning( _LW('Starting unauthenticated, please check' ' configuration')) if CONF.processing.store_data == 'none': LOG.warning( _LW('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not the ' 'desired behavior')) elif CONF.processing.store_data == 'swift': LOG.info( _LI('Introspection data will be stored in Swift in the ' 'container %s'), CONF.swift.container) db.init() try: hooks = [ext.name for ext in plugins_base.processing_hooks_manager()] except KeyError as exc: # stevedore raises KeyError on missing hook LOG.critical(_LC('Hook %s failed to load or was not found'), str(exc)) sys.exit(1) LOG.info(_LI('Enabled processing hooks: %s'), hooks) if CONF.firewall.manage_firewall: firewall.init() period = CONF.firewall.firewall_update_period utils.spawn_n(periodic_update, period) if CONF.timeout > 0: period = CONF.clean_up_period utils.spawn_n(periodic_clean_up, period) else: LOG.warning(_LW('Timeout is disabled in configuration'))
def before_update(self, introspection_data, node_info, **kwargs): if 'pci_devices' not in introspection_data: if CONF.pci_devices.alias: LOG.warning(_LW('No PCI devices information was received from ' 'the ramdisk.')) return alias_count = {self.aliases[id_pair]: count for id_pair, count in self._found_pci_devices_count( introspection_data['pci_devices']).items()} if alias_count: node_info.update_capabilities(**alias_count) LOG.info(_LI('Found the following PCI devices: %s'), alias_count)
def delete(uuid): """Delete a rule by its UUID.""" with db.ensure_transaction() as session: db.model_query(db.RuleAction, session=session).filter_by(rule=uuid).delete() db.model_query(db.RuleCondition, session=session) .filter_by(rule=uuid).delete() count = (db.model_query(db.Rule, session=session) .filter_by(uuid=uuid).delete()) if not count: raise utils.Error(_('Rule %s was not found') % uuid, code=404) LOG.info(_LI('Introspection rule %s was deleted'), uuid)
def shutdown(self): LOG.debug('Shutting down') firewall.clean_up() if self._periodics_worker is not None: self._periodics_worker.stop() self._periodics_worker.wait() self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) LOG.info(_LI('Shut down successfully'))
def _finish(ironic, node_info): LOG.debug('Forcing power off of node %s', node_info.uuid) try: ironic.node.set_power_state(node_info.uuid, 'off') except Exception as exc: msg = (_('Failed to power off node %(node)s, check it\'s power ' 'management configuration: %(exc)s') % {'node': node_info.uuid, 'exc': exc}) node_info.finished(error=msg) raise utils.Error(msg) node_info.finished() LOG.info(_LI('Introspection finished successfully for node %s'), node_info.uuid)
def before_update(self, introspection_data, node_info, **kwargs): if 'pci_devices' not in introspection_data: if CONF.pci_devices.alias: LOG.warning( _LW('No PCI devices information was received from ' 'the ramdisk.')) return alias_count = { self.aliases[id_pair]: count for id_pair, count in self._found_pci_devices_count( introspection_data['pci_devices']).items() } if alias_count: node_info.update_capabilities(**alias_count) LOG.info(_LI('Found the following PCI devices: %s'), alias_count)
def init(): if utils.get_auth_strategy() != 'noauth': utils.add_auth_middleware(app) else: LOG.warning(_LW('Starting unauthenticated, please check' ' configuration')) if CONF.processing.store_data == 'none': LOG.warning(_LW('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not the ' 'desired behavior')) elif CONF.processing.store_data == 'swift': LOG.info(_LI('Introspection data will be stored in Swift in the ' 'container %s'), CONF.swift.container) db.init() try: hooks = [ext.name for ext in plugins_base.processing_hooks_manager()] except KeyError as exc: # stevedore raises KeyError on missing hook LOG.critical(_LC('Hook %s failed to load or was not found'), str(exc)) sys.exit(1) LOG.info(_LI('Enabled processing hooks: %s'), hooks) if CONF.firewall.manage_firewall: firewall.init() period = CONF.firewall.firewall_update_period utils.spawn_n(periodic_update, period) if CONF.timeout > 0: period = CONF.clean_up_period utils.spawn_n(periodic_clean_up, period) else: LOG.warning(_LW('Timeout is disabled in configuration'))
def _process_node(node, introspection_data, node_info): # NOTE(dtantsur): repeat the check in case something changed utils.check_provision_state(node) node_info.create_ports(introspection_data.get('macs') or ()) _run_post_hooks(node_info, introspection_data) if CONF.processing.store_data == 'swift': swift_object_name = swift.store_introspection_data( introspection_data, node_info.uuid) LOG.info( _LI('Introspection data for node %(node)s was stored in ' 'Swift in object %(obj)s'), { 'node': node_info.uuid, 'obj': swift_object_name }) if CONF.processing.store_data_location: node_info.patch([{ 'op': 'add', 'path': '/extra/%s' % CONF.processing.store_data_location, 'value': swift_object_name }]) else: LOG.debug( 'Swift support is disabled, introspection data for node %s ' 'won\'t be stored', node_info.uuid) ironic = utils.get_client() firewall.update_filters(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.uuid} if node_info.options.get('new_ipmi_credentials'): new_username, new_password = ( node_info.options.get('new_ipmi_credentials')) utils.spawn_n(_finish_set_ipmi_credentials, ironic, node, node_info, introspection_data, new_username, new_password) resp['ipmi_setup_credentials'] = True resp['ipmi_username'] = new_username resp['ipmi_password'] = new_password else: utils.spawn_n(_finish, ironic, node_info) return resp
def _finish(ironic, node_info): LOG.debug('Forcing power off of node %s', node_info.uuid) try: ironic.node.set_power_state(node_info.uuid, 'off') except Exception as exc: msg = (_('Failed to power off node %(node)s, check it\'s power ' 'management configuration: %(exc)s') % { 'node': node_info.uuid, 'exc': exc }) node_info.finished(error=msg) raise utils.Error(msg) node_info.finished() LOG.info(_LI('Introspection finished successfully for node %s'), node_info.uuid)
def _process_node(node, introspection_data, node_info): # NOTE(dtantsur): repeat the check in case something changed ir_utils.check_provision_state(node) node_info.create_ports(introspection_data.get('macs') or ()) _run_post_hooks(node_info, introspection_data) if CONF.processing.store_data == 'swift': stored_data = {k: v for k, v in introspection_data.items() if k not in _STORAGE_EXCLUDED_KEYS} swift_object_name = swift.store_introspection_data(stored_data, node_info.uuid) LOG.info(_LI('Introspection data was stored in Swift in object %s'), swift_object_name, node_info=node_info, data=introspection_data) if CONF.processing.store_data_location: node_info.patch([{'op': 'add', 'path': '/extra/%s' % CONF.processing.store_data_location, 'value': swift_object_name}]) else: LOG.debug('Swift support is disabled, introspection data ' 'won\'t be stored', node_info=node_info, data=introspection_data) ironic = ir_utils.get_client() firewall.update_filters(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.uuid} if node_info.options.get('new_ipmi_credentials'): new_username, new_password = ( node_info.options.get('new_ipmi_credentials')) utils.executor().submit(_finish_set_ipmi_credentials, ironic, node, node_info, introspection_data, new_username, new_password) resp['ipmi_setup_credentials'] = True resp['ipmi_username'] = new_username resp['ipmi_password'] = new_password else: utils.executor().submit(_finish, ironic, node_info, introspection_data) return resp
def _store_data(node_info, data, suffix=None): if CONF.processing.store_data != 'swift': LOG.debug("Swift support is disabled, introspection data " "won't be stored", node_info=node_info) return swift_object_name = swift.store_introspection_data( _filter_data_excluded_keys(data), node_info.uuid, suffix=suffix ) LOG.info(_LI('Introspection data was stored in Swift in object ' '%s'), swift_object_name, node_info=node_info) if CONF.processing.store_data_location: node_info.patch([{'op': 'add', 'path': '/extra/%s' % CONF.processing.store_data_location, 'value': swift_object_name}])
def _detect_cpu_flags(self, inventory, node_info, data=None): flags = inventory['cpu'].get('flags') if not flags: LOG.warning(_LW('No CPU flags available, please update your ' 'introspection ramdisk'), data=data, node_info=node_info) return {} flags = set(flags) caps = {} for flag, name in CONF.capabilities.cpu_flags.items(): if flag in flags: caps[name] = 'true' LOG.info(_LI('CPU capabilities: %s'), list(caps), data=data, node_info=node_info) return caps
def before_update(self, introspection_data, node_info, **kwargs): """Update node with scheduler properties.""" inventory = utils.get_inventory(introspection_data, node_info=node_info) errors = [] root_disk = introspection_data.get('root_disk') if root_disk: introspection_data['local_gb'] = root_disk['size'] // units.Gi if CONF.processing.disk_partitioning_spacing: introspection_data['local_gb'] -= 1 else: introspection_data['local_gb'] = 0 try: introspection_data['cpus'] = int(inventory['cpu']['count']) introspection_data['cpu_arch'] = six.text_type( inventory['cpu']['architecture']) except (KeyError, ValueError, TypeError): errors.append(_('malformed or missing CPU information: %s') % inventory.get('cpu')) try: introspection_data['memory_mb'] = int( inventory['memory']['physical_mb']) except (KeyError, ValueError, TypeError): errors.append(_('malformed or missing memory information: %s; ' 'introspection requires physical memory size ' 'from dmidecode') % inventory.get('memory')) if errors: raise utils.Error(_('The following problems encountered: %s') % '; '.join(errors), node_info=node_info, data=introspection_data) LOG.info(_LI('Discovered data: CPUs: %(cpus)s %(cpu_arch)s, ' 'memory %(memory_mb)s MiB, disk %(local_gb)s GiB'), {key: introspection_data.get(key) for key in self.KEYS}, node_info=node_info, data=introspection_data) overwrite = CONF.processing.overwrite_existing properties = {key: str(introspection_data[key]) for key in self.KEYS if overwrite or not node_info.node().properties.get(key)} node_info.update_properties(**properties)
def _reapply(node_info): # runs in background try: introspection_data = _get_unprocessed_data(node_info.uuid) except Exception as exc: LOG.exception(_LE('Encountered exception while fetching ' 'stored introspection data'), node_info=node_info) msg = (_('Unexpected exception %(exc_class)s while fetching ' 'unprocessed introspection data from Swift: %(error)s') % { 'exc_class': exc.__class__.__name__, 'error': exc }) node_info.finished(error=msg) return failures = [] _run_pre_hooks(introspection_data, failures) if failures: LOG.error(_LE('Pre-processing failures detected reapplying ' 'introspection on stored data:\n%s'), '\n'.join(failures), node_info=node_info) node_info.finished(error='\n'.join(failures)) return try: ironic = ir_utils.get_client() node_info.create_ports(introspection_data.get('macs') or ()) _run_post_hooks(node_info, introspection_data) _store_data(node_info, introspection_data) node_info.invalidate_cache() rules.apply(node_info, introspection_data) _finish(ironic, node_info, introspection_data, power_off=False) except Exception as exc: LOG.exception(_LE('Encountered exception reapplying ' 'introspection on stored data'), node_info=node_info, data=introspection_data) node_info.finished(error=str(exc)) else: LOG.info(_LI('Successfully reapplied introspection on stored ' 'data'), node_info=node_info, data=introspection_data)
def _store_logs(self, logs, introspection_data): if not CONF.processing.ramdisk_logs_dir: LOG.warning( _LW('Failed to store logs received from the ramdisk ' 'because ramdisk_logs_dir configuration option ' 'is not set')) return if not os.path.exists(CONF.processing.ramdisk_logs_dir): os.makedirs(CONF.processing.ramdisk_logs_dir) time_fmt = datetime.datetime.utcnow().strftime(self.DATETIME_FORMAT) bmc_address = introspection_data.get('ipmi_address', 'unknown') file_name = 'bmc_%s_%s' % (bmc_address, time_fmt) with open(os.path.join(CONF.processing.ramdisk_logs_dir, file_name), 'wb') as fp: fp.write(base64.b64decode(logs)) LOG.info(_LI('Ramdisk logs stored in file %s'), file_name)
def before_update(self, introspection_data, node_info, **kwargs): current_devices = self._get_serials(introspection_data) if not current_devices: LOG.warning(_LW('No block device was received from ramdisk'), node_info=node_info, data=introspection_data) return node = node_info.node() if 'root_device' in node.properties: LOG.info(_LI('Root device is already known for the node'), node_info=node_info, data=introspection_data) return if 'block_devices' in node.extra: # Compare previously discovered devices with the current ones previous_devices = node.extra['block_devices']['serials'] new_devices = [device for device in current_devices if device not in previous_devices] if len(new_devices) > 1: LOG.warning(_LW('Root device cannot be identified because ' 'multiple new devices were found'), node_info=node_info, data=introspection_data) return elif len(new_devices) == 0: LOG.warning(_LW('No new devices were found'), node_info=node_info, data=introspection_data) return node_info.patch([ {'op': 'remove', 'path': '/extra/block_devices'}, {'op': 'add', 'path': '/properties/root_device', 'value': {'serial': new_devices[0]}} ]) else: # No previously discovered devices - save the inspector block # devices in node.extra node_info.patch([{'op': 'add', 'path': '/extra/block_devices', 'value': {'serials': current_devices}}])
def _process_node(node, introspection_data, node_info): # NOTE(dtantsur): repeat the check in case something changed utils.check_provision_state(node) node_info.create_ports(introspection_data.get('macs') or ()) _run_post_hooks(node_info, introspection_data) if CONF.processing.store_data == 'swift': swift_object_name = swift.store_introspection_data(introspection_data, node_info.uuid) LOG.info(_LI('Introspection data for node %(node)s was stored in ' 'Swift in object %(obj)s'), {'node': node_info.uuid, 'obj': swift_object_name}) if CONF.processing.store_data_location: node_info.patch([{'op': 'add', 'path': '/extra/%s' % CONF.processing.store_data_location, 'value': swift_object_name}]) else: LOG.debug('Swift support is disabled, introspection data for node %s ' 'won\'t be stored', node_info.uuid) ironic = utils.get_client() firewall.update_filters(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.uuid} if node_info.options.get('new_ipmi_credentials'): new_username, new_password = ( node_info.options.get('new_ipmi_credentials')) utils.spawn_n(_finish_set_ipmi_credentials, ironic, node, node_info, introspection_data, new_username, new_password) resp['ipmi_setup_credentials'] = True resp['ipmi_username'] = new_username resp['ipmi_password'] = new_password else: utils.spawn_n(_finish, ironic, node_info) return resp
def get_inventory(data, node_info=None): """Get and validate the hardware inventory from introspection data.""" inventory = data.get('inventory') # TODO(dtantsur): validate inventory using JSON schema if not inventory: raise Error(_('Hardware inventory is empty or missing'), data=data, node_info=node_info) for key in _INVENTORY_MANDATORY_KEYS: if not inventory.get(key): raise Error(_('Invalid hardware inventory: %s key is missing ' 'or empty') % key, data=data, node_info=node_info) if not inventory.get('disks'): LOG.info(_LI('No disks were detected in the inventory, assuming this ' 'is a disk-less node'), data=data, node_info=node_info) # Make sure the code iterating over it does not fail with a TypeError inventory['disks'] = [] return inventory
def create_node(driver, ironic=None, **attributes): """Create ironic node and cache it. * Create new node in ironic. * Cache it in inspector. :param driver: driver for Ironic node. :param ironic: ronic client instance. :param attributes: dict, additional keyword arguments to pass to the ironic client on node creation. :return: NodeInfo, or None in case error happened. """ if ironic is None: ironic = ir_utils.get_client() try: node = ironic.node.create(driver=driver, **attributes) except exceptions.InvalidAttribute as e: LOG.error(_LE('Failed to create new node: %s'), e) else: LOG.info(_LI('Node %s was created successfully'), node.uuid) return add_node(node.uuid, ironic=ironic)
def before_update(self, introspection_data, node_info, **kwargs): """Drop ports that are not present in the data.""" if CONF.processing.keep_ports == 'present': expected_macs = { iface['mac'] for iface in introspection_data['all_interfaces'].values() } elif CONF.processing.keep_ports == 'added': expected_macs = set(introspection_data['macs']) else: return # list is required as we modify underlying dict for port in list(node_info.ports().values()): if port.address not in expected_macs: LOG.info(_LI("Deleting port %(port)s as its MAC %(mac)s is " "not in expected MAC list %(expected)s"), {'port': port.uuid, 'mac': port.address, 'expected': list(sorted(expected_macs))}, node_info=node_info, data=introspection_data) node_info.delete_port(port)