def lookup_node(self, hardware_info, timeout, starting_interval): timer = backoff.BackOffLoopingCall(self._do_lookup, hardware_info=hardware_info) try: node_content = timer.start(starting_interval=starting_interval, timeout=timeout).wait() except backoff.LoopingCallTimeOut: raise errors.LookupNodeError('Could not look up node info. Check ' 'logs for details.') return node_content
def lookup_node(self, hardware_info, timeout, starting_interval, node_uuid=None, max_interval=30): retry = tenacity.retry( retry=tenacity.retry_if_result(lambda r: r is False), stop=tenacity.stop_after_delay(timeout), wait=tenacity.wait_random_exponential(min=starting_interval, max=max_interval), reraise=True) try: return retry(self._do_lookup)(hardware_info=hardware_info, node_uuid=node_uuid) except tenacity.RetryError: raise errors.LookupNodeError('Could not look up node info. Check ' 'logs for details.')
def test_error_classes(self): cases = [ (errors.InvalidContentError(DETAILS), SAME_DETAILS), (errors.NotFound(), SAME_CL_DETAILS), (errors.CommandExecutionError(DETAILS), SAME_DETAILS), (errors.InvalidCommandError(DETAILS), SAME_DETAILS), (errors.InvalidCommandParamsError(DETAILS), SAME_DETAILS), (errors.RequestedObjectNotFoundError('type_descr', 'obj_id'), DIFF_CL_DETAILS), (errors.IronicAPIError(DETAILS), SAME_DETAILS), (errors.HeartbeatError(DETAILS), SAME_DETAILS), (errors.LookupNodeError(DETAILS), SAME_DETAILS), (errors.LookupAgentIPError(DETAILS), SAME_DETAILS), (errors.LookupAgentInterfaceError(DETAILS), SAME_DETAILS), (errors.ImageDownloadError('image_id', DETAILS), DIFF_CL_DETAILS), (errors.ImageChecksumError('image_id', '/foo/image_id', 'incorrect', 'correct'), DIFF_CL_DETAILS), (errors.ImageWriteError('device', 'exit_code', 'stdout', 'stderr'), DIFF_CL_DETAILS), (errors.ConfigDriveTooLargeError('filename', 'filesize'), DIFF_CL_DETAILS), (errors.ConfigDriveWriteError('device', 'exit_code', 'stdout', 'stderr'), DIFF_CL_DETAILS), (errors.SystemRebootError('exit_code', 'stdout', 'stderr'), DIFF_CL_DETAILS), (errors.BlockDeviceEraseError(DETAILS), SAME_DETAILS), (errors.BlockDeviceError(DETAILS), SAME_DETAILS), (errors.VirtualMediaBootError(DETAILS), SAME_DETAILS), (errors.UnknownNodeError(), DEFAULT_DETAILS), (errors.UnknownNodeError(DETAILS), SAME_DETAILS), (errors.HardwareManagerNotFound(), DEFAULT_DETAILS), (errors.HardwareManagerNotFound(DETAILS), SAME_DETAILS), (errors.HardwareManagerMethodNotFound('method'), DIFF_CL_DETAILS), (errors.IncompatibleHardwareMethodError(), DEFAULT_DETAILS), (errors.IncompatibleHardwareMethodError(DETAILS), SAME_DETAILS), ] for (obj, check_details) in cases: self._test_class(obj, check_details)
def _do_lookup(self, hardware_info, node_uuid): """The actual call to lookup a node.""" params = { 'addresses': ','.join(iface.mac_address for iface in hardware_info['interfaces'] if iface.mac_address) } if node_uuid: params['node_uuid'] = node_uuid LOG.debug('Looking up node with addresses %r and UUID %s at %s', params['addresses'], node_uuid, self.api_url) try: response = self._request( 'GET', self.lookup_api, headers=self._get_ironic_api_version_header(), params=params) except (requests.exceptions.Timeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout, requests.exceptions.HTTPError) as err: LOG.warning( 'Error detected while attempting to perform lookup ' 'with %s, retrying. Error: %s', self.api_url, err) return False except Exception as err: # NOTE(TheJulia): If you're looking here, and you're wondering # why the retry logic is not working or your investigating a weird # error or even IPA just exiting, # See https://storyboard.openstack.org/#!/story/2007968 # To be clear, we're going to try to provide as much detail as # possible in the exit handling msg = ('Unhandled error looking up node with addresses {} at ' '{}: {}'.format(params['addresses'], self.api_url, err)) # No matter what we do at this point, IPA is going to exit. # This is because we don't know why the exception occured and # we likely should not try to retry as such. # We will attempt to provide as much detail to the logs as # possible as to what occured, although depending on the logging # subsystem, additional errors can occur, thus the additional # handling below. try: LOG.exception(msg) return False except Exception as exc_err: LOG.error(msg) exc_msg = ('Unexpected exception occured while trying to ' 'log additional detail. Error: {}'.format(exc_err)) LOG.error(exc_msg) raise errors.LookupNodeError(msg) if response.status_code != requests.codes.OK: LOG.warning( 'Failed looking up node with addresses %r at %s. ' '%s. Check if inspection has completed.', params['addresses'], self.api_url, self._error_from_response(response)) return False try: content = jsonutils.loads(response.content) except Exception as e: LOG.warning('Error decoding response: %s', e) return False # Check for valid response data if 'node' not in content or 'uuid' not in content['node']: LOG.warning( 'Got invalid node data in response to query for node ' 'with addresses %r from %s: %s', params['addresses'], self.api_url, content, ) return False if 'config' not in content: # Old API try: content['config'] = { 'heartbeat_timeout': content.pop('heartbeat_timeout') } except KeyError: LOG.warning('Got invalid heartbeat from the API: %s', content) return False # Got valid content return content