def lookup_node(self, hardware_info, timeout, starting_interval):
timer = backoff.BackOffLoopingCall(self._do_lookup,
hardware_info=hardware_info)
try:
node_content = timer.start(starting_interval=starting_interval,
timeout=timeout).wait()
except backoff.LoopingCallTimeOut:
raise errors.LookupNodeError('Could not look up node info. Check '
'logs for details.')
return node_content
def lookup_node(self, hardware_info, timeout, starting_interval,
node_uuid=None, max_interval=30):
retry = tenacity.retry(
retry=tenacity.retry_if_result(lambda r: r is False),
stop=tenacity.stop_after_delay(timeout),
wait=tenacity.wait_random_exponential(min=starting_interval,
max=max_interval),
reraise=True)
try:
return retry(self._do_lookup)(hardware_info=hardware_info,
node_uuid=node_uuid)
except tenacity.RetryError:
raise errors.LookupNodeError('Could not look up node info. Check '
'logs for details.')
def test_error_classes(self):
cases = [
(errors.InvalidContentError(DETAILS), SAME_DETAILS),
(errors.NotFound(), SAME_CL_DETAILS),
(errors.CommandExecutionError(DETAILS), SAME_DETAILS),
(errors.InvalidCommandError(DETAILS), SAME_DETAILS),
(errors.InvalidCommandParamsError(DETAILS), SAME_DETAILS),
(errors.RequestedObjectNotFoundError('type_descr',
'obj_id'), DIFF_CL_DETAILS),
(errors.IronicAPIError(DETAILS), SAME_DETAILS),
(errors.HeartbeatError(DETAILS), SAME_DETAILS),
(errors.LookupNodeError(DETAILS), SAME_DETAILS),
(errors.LookupAgentIPError(DETAILS), SAME_DETAILS),
(errors.LookupAgentInterfaceError(DETAILS), SAME_DETAILS),
(errors.ImageDownloadError('image_id', DETAILS), DIFF_CL_DETAILS),
(errors.ImageChecksumError('image_id', '/foo/image_id',
'incorrect',
'correct'), DIFF_CL_DETAILS),
(errors.ImageWriteError('device', 'exit_code', 'stdout',
'stderr'), DIFF_CL_DETAILS),
(errors.ConfigDriveTooLargeError('filename',
'filesize'), DIFF_CL_DETAILS),
(errors.ConfigDriveWriteError('device', 'exit_code', 'stdout',
'stderr'), DIFF_CL_DETAILS),
(errors.SystemRebootError('exit_code', 'stdout',
'stderr'), DIFF_CL_DETAILS),
(errors.BlockDeviceEraseError(DETAILS), SAME_DETAILS),
(errors.BlockDeviceError(DETAILS), SAME_DETAILS),
(errors.VirtualMediaBootError(DETAILS), SAME_DETAILS),
(errors.UnknownNodeError(), DEFAULT_DETAILS),
(errors.UnknownNodeError(DETAILS), SAME_DETAILS),
(errors.HardwareManagerNotFound(), DEFAULT_DETAILS),
(errors.HardwareManagerNotFound(DETAILS), SAME_DETAILS),
(errors.HardwareManagerMethodNotFound('method'), DIFF_CL_DETAILS),
(errors.IncompatibleHardwareMethodError(), DEFAULT_DETAILS),
(errors.IncompatibleHardwareMethodError(DETAILS), SAME_DETAILS),
]
for (obj, check_details) in cases:
self._test_class(obj, check_details)
def _do_lookup(self, hardware_info, node_uuid):
"""The actual call to lookup a node."""
params = {
'addresses':
','.join(iface.mac_address for iface in hardware_info['interfaces']
if iface.mac_address)
}
if node_uuid:
params['node_uuid'] = node_uuid
LOG.debug('Looking up node with addresses %r and UUID %s at %s',
params['addresses'], node_uuid, self.api_url)
try:
response = self._request(
'GET',
self.lookup_api,
headers=self._get_ironic_api_version_header(),
params=params)
except (requests.exceptions.Timeout,
requests.exceptions.ConnectTimeout,
requests.exceptions.ConnectionError,
requests.exceptions.ReadTimeout,
requests.exceptions.HTTPError) as err:
LOG.warning(
'Error detected while attempting to perform lookup '
'with %s, retrying. Error: %s', self.api_url, err)
return False
except Exception as err:
# NOTE(TheJulia): If you're looking here, and you're wondering
# why the retry logic is not working or your investigating a weird
# error or even IPA just exiting,
# See https://storyboard.openstack.org/#!/story/2007968
# To be clear, we're going to try to provide as much detail as
# possible in the exit handling
msg = ('Unhandled error looking up node with addresses {} at '
'{}: {}'.format(params['addresses'], self.api_url, err))
# No matter what we do at this point, IPA is going to exit.
# This is because we don't know why the exception occured and
# we likely should not try to retry as such.
# We will attempt to provide as much detail to the logs as
# possible as to what occured, although depending on the logging
# subsystem, additional errors can occur, thus the additional
# handling below.
try:
LOG.exception(msg)
return False
except Exception as exc_err:
LOG.error(msg)
exc_msg = ('Unexpected exception occured while trying to '
'log additional detail. Error: {}'.format(exc_err))
LOG.error(exc_msg)
raise errors.LookupNodeError(msg)
if response.status_code != requests.codes.OK:
LOG.warning(
'Failed looking up node with addresses %r at %s. '
'%s. Check if inspection has completed.', params['addresses'],
self.api_url, self._error_from_response(response))
return False
try:
content = jsonutils.loads(response.content)
except Exception as e:
LOG.warning('Error decoding response: %s', e)
return False
# Check for valid response data
if 'node' not in content or 'uuid' not in content['node']:
LOG.warning(
'Got invalid node data in response to query for node '
'with addresses %r from %s: %s',
params['addresses'],
self.api_url,
content,
)
return False
if 'config' not in content:
# Old API
try:
content['config'] = {
'heartbeat_timeout': content.pop('heartbeat_timeout')
}
except KeyError:
LOG.warning('Got invalid heartbeat from the API: %s', content)
return False
# Got valid content
return content
