def _finish_set_ipmi_credentials(ironic, node, cached_node, node_info, new_username, new_password): patch = [{'op': 'add', 'path': '/driver_info/ipmi_username', 'value': new_username}, {'op': 'add', 'path': '/driver_info/ipmi_password', 'value': new_password}] if not utils.get_ipmi_address(node) and node_info.get('ipmi_address'): patch.append({'op': 'add', 'path': '/driver_info/ipmi_address', 'value': node_info['ipmi_address']}) utils.retry_on_conflict(ironic.node.update, cached_node.uuid, patch) for attempt in range(_CREDENTIALS_WAIT_RETRIES): try: # We use this call because it requires valid credentials. # We don't care about boot device, obviously. ironic.node.get_boot_device(cached_node.uuid) except Exception as exc: LOG.info(_LI('Waiting for credentials update on node %(node)s,' ' attempt %(attempt)d current error is %(exc)s') % {'node': cached_node.uuid, 'attempt': attempt, 'exc': exc}) eventlet.greenthread.sleep(_CREDENTIALS_WAIT_PERIOD) else: _finish(ironic, cached_node) return msg = (_('Failed to validate updated IPMI credentials for node ' '%s, node might require maintenance') % cached_node.uuid) cached_node.finished(error=msg) raise utils.Error(msg)
def _prepare_for_pxe(ironic, cached_node, macs, setup_ipmi_credentials): if macs: LOG.info('Whitelisting MAC\'s %s for node %s on the firewall', macs, cached_node.uuid) firewall.update_filters(ironic) if not setup_ipmi_credentials: try: utils.retry_on_conflict(ironic.node.set_boot_device, cached_node.uuid, 'pxe', persistent=False) except Exception as exc: LOG.warning('Failed to set boot device to PXE for node %s: %s', cached_node.uuid, exc) try: utils.retry_on_conflict(ironic.node.set_power_state, cached_node.uuid, 'reboot') except Exception as exc: raise utils.Error('Failed to power on node %s, check it\'s power ' 'management configuration:\n%s' % (cached_node.uuid, exc)) else: LOG.info('Introspection environment is ready for node %s, ' 'manual power on is required within %d seconds', cached_node.uuid, conf.getint('discoverd', 'timeout'))
def _prepare_for_pxe(ironic, cached_node, macs, setup_ipmi_credentials): if macs: LOG.info('Whitelisting MAC\'s %s for node %s on the firewall', macs, cached_node.uuid) firewall.update_filters(ironic) if not setup_ipmi_credentials: try: utils.retry_on_conflict(ironic.node.set_boot_device, cached_node.uuid, 'pxe', persistent=False) except Exception as exc: LOG.warning('Failed to set boot device to PXE for node %s: %s', cached_node.uuid, exc) try: utils.retry_on_conflict(ironic.node.set_power_state, cached_node.uuid, 'reboot') except Exception as exc: raise utils.Error('Failed to power on node %s, check it\'s power ' 'management configuration:\n%s' % (cached_node.uuid, exc)) else: LOG.info('Introspection environment is ready for node %s, ' 'manual power on is required within %d seconds', cached_node.uuid, conf.getint('discoverd', 'timeout'))
def _finish(ironic, cached_node): _force_power_off(ironic, cached_node) patch = [{ 'op': 'add', 'path': '/extra/newly_discovered', 'value': 'true' }, { 'op': 'remove', 'path': '/extra/on_discovery' }] utils.retry_on_conflict(ironic.node.update, cached_node.uuid, patch) cached_node.finished() LOG.info('Introspection finished successfully for node %s', cached_node.uuid)
def test_retry_on_conflict(self): call = mock.Mock() call.side_effect = ([exceptions.Conflict()] * (utils.RETRY_COUNT - 1) + [mock.sentinel.result]) res = utils.retry_on_conflict(call, 1, 2, x=3) self.assertEqual(mock.sentinel.result, res) call.assert_called_with(1, 2, x=3) self.assertEqual(utils.RETRY_COUNT, call.call_count)
def test_retry_on_conflict(self): call = mock.Mock() call.side_effect = ([exceptions.Conflict()] * (utils.RETRY_COUNT - 1) + [mock.sentinel.result]) res = utils.retry_on_conflict(call, 1, 2, x=3) self.assertEqual(mock.sentinel.result, res) call.assert_called_with(1, 2, x=3) self.assertEqual(utils.RETRY_COUNT, call.call_count)
def _background_start_discover(ironic, node, setup_ipmi_credentials): patch = [{'op': 'add', 'path': '/extra/on_discovery', 'value': 'true'}] utils.retry_on_conflict(ironic.node.update, node.uuid, patch) # TODO(dtantsur): pagination macs = [p.address for p in ironic.node.list_ports(node.uuid, limit=0)] cached_node = node_cache.add_node(node.uuid, bmc_address=_get_ipmi_address(node), mac=macs) cached_node.set_option('setup_ipmi_credentials', setup_ipmi_credentials) try: _prepare_for_pxe(ironic, cached_node, macs, setup_ipmi_credentials) except utils.Error as exc: cached_node.finished(error=str(exc)) except Exception as exc: msg = 'Unexpected exception during preparing for PXE boot' LOG.exception(msg) cached_node.finished(error=msg)
def _background_start_discover(ironic, node, setup_ipmi_credentials): patch = [{'op': 'add', 'path': '/extra/on_discovery', 'value': 'true'}] utils.retry_on_conflict(ironic.node.update, node.uuid, patch) # TODO(dtantsur): pagination macs = [p.address for p in ironic.node.list_ports(node.uuid, limit=0)] cached_node = node_cache.add_node( node.uuid, bmc_address=node.driver_info.get('ipmi_address'), mac=macs) cached_node.set_option('setup_ipmi_credentials', setup_ipmi_credentials) try: _prepare_for_pxe(ironic, cached_node, macs, setup_ipmi_credentials) except utils.Error as exc: cached_node.finished(error=str(exc)) except Exception as exc: msg = 'Unexpected exception during preparing for PXE boot' LOG.exception(msg) cached_node.finished(error=msg)
def _finish(ironic, cached_node): LOG.debug('Forcing power off of node %s', cached_node.uuid) try: utils.retry_on_conflict(ironic.node.set_power_state, cached_node.uuid, 'off') except Exception as exc: msg = (_('Failed to power off node %(node)s, check it\'s power ' 'management configuration: %(exc)s') % {'node': cached_node.uuid, 'exc': exc}) cached_node.finished(error=msg) raise utils.Error(msg) cached_node.finished() patch = [{'op': 'add', 'path': '/extra/newly_discovered', 'value': 'true'}, {'op': 'remove', 'path': '/extra/on_discovery'}] utils.retry_on_conflict(ironic.node.update, cached_node.uuid, patch) LOG.info(_LI('Introspection finished successfully for node %s'), cached_node.uuid)
def _process_node(ironic, node, node_info, cached_node): # NOTE(dtantsur): repeat the check in case something changed utils.check_provision_state(node) ports = {} for mac in (node_info.get('macs') or ()): try: port = ironic.port.create(node_uuid=node.uuid, address=mac) ports[mac] = port except exceptions.Conflict: LOG.warning(_LW('MAC %(mac)s appeared in introspection data for ' 'node %(node)s, but already exists in ' 'database - skipping') % {'mac': mac, 'node': node.uuid}) node_patches, port_patches = _run_post_hooks(node, ports, node_info) # Invalidate cache in case of hooks modifying options cached_node.invalidate_cache() node = utils.retry_on_conflict(ironic.node.update, node.uuid, node_patches) for mac, patches in port_patches.items(): utils.retry_on_conflict(ironic.port.update, ports[mac].uuid, patches) LOG.debug('Node %s was updated with data from introspection process, ' 'patches %s, port patches %s', node.uuid, node_patches, port_patches) firewall.update_filters(ironic) if cached_node.options.get('new_ipmi_credentials'): new_username, new_password = ( cached_node.options.get('new_ipmi_credentials')) eventlet.greenthread.spawn_n(_finish_set_ipmi_credentials, ironic, node, cached_node, node_info, new_username, new_password) return {'ipmi_setup_credentials': True, 'ipmi_username': new_username, 'ipmi_password': new_password} else: eventlet.greenthread.spawn_n(_finish, ironic, cached_node) return {}
def _process_node(ironic, node, node_info, cached_node): ports = {} for mac in (node_info.get('macs') or ()): try: port = ironic.port.create(node_uuid=node.uuid, address=mac) ports[mac] = port except exceptions.Conflict: LOG.warning( 'MAC %(mac)s appeared in introspection data for ' 'node %(node)s, but already exists in ' 'database - skipping', { 'mac': mac, 'node': node.uuid }) node_patches, port_patches = _run_post_hooks(node, ports, node_info) node = utils.retry_on_conflict(ironic.node.update, node.uuid, node_patches) for mac, patches in port_patches.items(): utils.retry_on_conflict(ironic.port.update, ports[mac].uuid, patches) LOG.debug( 'Node %s was updated with data from introspection process, ' 'patches %s, port patches %s', node.uuid, node_patches, port_patches) firewall.update_filters(ironic) if cached_node.options.get('setup_ipmi_credentials'): eventlet.greenthread.spawn_n(_wait_for_power_management, ironic, cached_node) return { 'ipmi_setup_credentials': True, 'ipmi_username': node.driver_info.get('ipmi_username'), 'ipmi_password': node.driver_info.get('ipmi_password') } else: eventlet.greenthread.spawn_n(_finish, ironic, cached_node) return {}
def _force_power_off(ironic, cached_node): LOG.debug('Forcing power off of node %s', cached_node.uuid) try: utils.retry_on_conflict(ironic.node.set_power_state, cached_node.uuid, 'off') except Exception as exc: msg = ('Failed to power off node %s, check it\'s power ' 'management configuration: %s' % (cached_node.uuid, exc)) cached_node.finished(error=msg) raise utils.Error(msg) deadline = cached_node.started_at + conf.getint('discoverd', 'timeout') while time.time() < deadline: node = ironic.node.get(cached_node.uuid) if (node.power_state or '').lower() == 'power off': return LOG.info('Waiting for node %s to power off, current state is %s', cached_node.uuid, node.power_state) eventlet.greenthread.sleep(_POWER_OFF_CHECK_PERIOD) msg = ('Timeout waiting for node %s to power off after introspection' % cached_node.uuid) cached_node.finished(error=msg) raise utils.Error(msg)
def introspect(uuid, new_ipmi_credentials=None): """Initiate hardware properties introspection for a given node. :param uuid: node uuid :param new_ipmi_credentials: tuple (new username, new password) or None :raises: Error """ ironic = utils.get_client() try: node = ironic.node.get(uuid) except exceptions.NotFound: raise utils.Error(_("Cannot find node %s") % uuid, code=404) except exceptions.HttpError as exc: raise utils.Error(_("Cannot get node %(node)s: %(exc)s") % {'node': uuid, 'exc': exc}) utils.check_provision_state(node) if new_ipmi_credentials: new_ipmi_credentials = ( _validate_ipmi_credentials(node, new_ipmi_credentials)) else: validation = utils.retry_on_conflict(ironic.node.validate, node.uuid) if not validation.power['result']: msg = _('Failed validation of power interface for node %(node)s, ' 'reason: %(reason)s') raise utils.Error(msg % {'node': node.uuid, 'reason': validation.power['reason']}) cached_node = node_cache.add_node(node.uuid, bmc_address=utils.get_ipmi_address(node)) cached_node.set_option('new_ipmi_credentials', new_ipmi_credentials) def _handle_exceptions(): try: _background_introspect(ironic, cached_node) except utils.Error as exc: cached_node.finished(error=str(exc)) except Exception as exc: msg = _('Unexpected exception in background introspection thread') LOG.exception(msg) cached_node.finished(error=msg) eventlet.greenthread.spawn_n(_handle_exceptions)
def introspect(uuid, setup_ipmi_credentials=False): """Initiate hardware properties introspection for a given node. :param uuid: node uuid :raises: Error """ ironic = utils.get_client() try: node = ironic.node.get(uuid) except exceptions.NotFound: raise utils.Error("Cannot find node %s" % uuid, code=404) except exceptions.HttpError as exc: raise utils.Error("Cannot get node %s: %s" % (uuid, exc)) if (setup_ipmi_credentials and not conf.getboolean('discoverd', 'enable_setting_ipmi_credentials')): raise utils.Error( 'IPMI credentials setup is disabled in configuration') if not node.maintenance: provision_state = node.provision_state if provision_state and provision_state.lower() not in VALID_STATES: msg = ('Refusing to introspect node %s with provision state "%s" ' 'and maintenance mode off') raise utils.Error(msg % (node.uuid, provision_state)) power_state = node.power_state if power_state and power_state.lower() not in VALID_POWER_STATES: msg = ('Refusing to introspect node %s with power state "%s" ' 'and maintenance mode off') raise utils.Error(msg % (node.uuid, power_state)) else: LOG.info('Node %s is in maintenance mode, skipping power and provision' ' states check') if not setup_ipmi_credentials: validation = utils.retry_on_conflict(ironic.node.validate, node.uuid) if not validation.power['result']: msg = ('Failed validation of power interface for node %s, ' 'reason: %s') raise utils.Error(msg % (node.uuid, validation.power['reason'])) eventlet.greenthread.spawn_n(_background_start_discover, ironic, node, setup_ipmi_credentials=setup_ipmi_credentials)
def _wait_for_power_management(ironic, cached_node): deadline = cached_node.started_at + conf.getint('discoverd', 'timeout') while time.time() < deadline: eventlet.greenthread.sleep(_POWER_CHECK_PERIOD) validation = utils.retry_on_conflict(ironic.node.validate, cached_node.uuid) if validation.power['result']: _finish(ironic, cached_node) return LOG.debug( 'Waiting for management credentials on node %s ' 'to be updated, current error: %s', cached_node.uuid, validation.power['reason']) msg = ('Timeout waiting for power credentials update of node %s ' 'after introspection' % cached_node.uuid) LOG.error(msg) cached_node.finished(error=msg)
def introspect(uuid, setup_ipmi_credentials=False): """Initiate hardware properties introspection for a given node. :param uuid: node uuid :raises: Error """ ironic = utils.get_client() try: node = ironic.node.get(uuid) except exceptions.NotFound: raise utils.Error("Cannot find node %s" % uuid, code=404) except exceptions.HttpError as exc: raise utils.Error("Cannot get node %s: %s" % (uuid, exc)) if (setup_ipmi_credentials and not conf.getboolean('discoverd', 'enable_setting_ipmi_credentials')): raise utils.Error( 'IPMI credentials setup is disabled in configuration') if not node.maintenance: provision_state = node.provision_state if provision_state and provision_state.lower() not in VALID_STATES: msg = ('Refusing to introspect node %s with provision state "%s" ' 'and maintenance mode off') raise utils.Error(msg % (node.uuid, provision_state)) else: LOG.info('Node %s is in maintenance mode, skipping power and provision' ' states check') if not setup_ipmi_credentials: validation = utils.retry_on_conflict(ironic.node.validate, node.uuid) if not validation.power['result']: msg = ('Failed validation of power interface for node %s, ' 'reason: %s') raise utils.Error(msg % (node.uuid, validation.power['reason'])) eventlet.greenthread.spawn_n(_background_start_discover, ironic, node, setup_ipmi_credentials=setup_ipmi_credentials)
def _background_introspect(ironic, cached_node): patch = [{'op': 'add', 'path': '/extra/on_discovery', 'value': 'true'}] utils.retry_on_conflict(ironic.node.update, cached_node.uuid, patch) # TODO(dtantsur): pagination macs = [p.address for p in ironic.node.list_ports(cached_node.uuid, limit=0)] if macs: cached_node.add_attribute(node_cache.MACS_ATTRIBUTE, macs) LOG.info(_LI('Whitelisting MAC\'s %(macs)s for node %(node)s on the' ' firewall') % {'macs': macs, 'node': cached_node.uuid}) firewall.update_filters(ironic) if not cached_node.options.get('new_ipmi_credentials'): try: utils.retry_on_conflict(ironic.node.set_boot_device, cached_node.uuid, 'pxe', persistent=False) except Exception as exc: LOG.warning(_LW('Failed to set boot device to PXE for' ' node %(node)s: %(exc)s') % {'node': cached_node.uuid, 'exc': exc}) try: utils.retry_on_conflict(ironic.node.set_power_state, cached_node.uuid, 'reboot') except Exception as exc: raise utils.Error(_('Failed to power on node %(node)s,' ' check it\'s power ' 'management configuration:\n%(exc)s') % {'node': cached_node.uuid, 'exc': exc}) else: LOG.info(_LI('Introspection environment is ready for node %(node)s, ' 'manual power on is required within %(timeout)d seconds') % {'node': cached_node.uuid, 'timeout': CONF.discoverd.timeout})