def retry_request(self,
method,
action,
body=None,
headers=None,
params=None):
"""Call do_request with the default retry configuration.
Only idempotent requests should retry failed connection attempts.
:raises: ConnectionFailed if the maximum # of retries is exceeded
"""
max_attempts = self.retries + 1
for i in range(max_attempts):
try:
return self.do_request(method,
action,
body=body,
headers=headers,
params=params)
except exceptions.ConnectionFailed:
# Exception has already been logged by do_request()
if i < self.retries:
_logger.debug('Retrying connection to Neutron service')
time.sleep(self.retry_interval)
elif self.raise_errors:
raise
if self.retries:
msg = (_("Failed to connect to Neutron server after %d attempts") %
max_attempts)
else:
msg = _("Failed to connect Neutron server")
raise exceptions.ConnectionFailed(reason=msg)
def _cs_request(self, *args, **kwargs):
kargs = {}
kargs.setdefault('headers', kwargs.get('headers', {}))
kargs['headers']['User-Agent'] = self.USER_AGENT
if 'content_type' in kwargs:
kargs['headers']['Content-Type'] = kwargs['content_type']
kargs['headers']['Accept'] = kwargs['content_type']
else:
kargs['headers']['Content-Type'] = self.content_type
kargs['headers']['Accept'] = self.content_type
if 'body' in kwargs:
kargs['body'] = kwargs['body']
args = utils.safe_encode_list(args)
kargs = utils.safe_encode_dict(kargs)
utils.http_log_req(_logger, args, kargs)
try:
resp, body = self.request(*args, **kargs)
except Exception as e:
# Wrap the low-level connection error (socket timeout, redirect
# limit, decompression error, etc) into our custom high-level
# connection exception (it is excepted in the upper layers of code)
raise exceptions.ConnectionFailed(reason=e)
utils.http_log_resp(_logger, resp, body)
status_code = self.get_status_code(resp)
if status_code == 401:
raise exceptions.Unauthorized(message=body)
elif status_code == 403:
raise exceptions.Forbidden(message=body)
return resp, body
def _cs_request(self, *args, **kwargs):
kargs = {}
kargs.setdefault('headers', kwargs.get('headers', {}))
kargs['headers']['User-Agent'] = self.USER_AGENT
if 'body' in kwargs:
kargs['body'] = kwargs['body']
if self.log_credentials:
log_kargs = kargs
else:
log_kargs = self._strip_credentials(kargs)
utils.http_log_req(_logger, args, log_kargs)
try:
resp, body = self.request(*args, **kargs)
except requests.exceptions.SSLError as e:
raise exceptions.SslCertificateValidationError(reason=e)
except Exception as e:
# Wrap the low-level connection error (socket timeout, redirect
# limit, decompression error, etc) into our custom high-level
# connection exception (it is excepted in the upper layers of code)
_logger.debug("throwing ConnectionFailed : %s", e)
raise exceptions.ConnectionFailed(reason=e)
utils.http_log_resp(_logger, resp, body)
if resp.status_code == 401:
raise exceptions.Unauthorized(message=body)
return resp, body
def test_configure_tenant_networks_update_fail(self, client_mock):
client = client_mock.return_value
client.update_port.side_effect = neutron_exceptions.ConnectionFailed(
reason='meow')
with task_manager.acquire(self.context, self.node.id) as task:
self.assertRaisesRegex(
exception.NetworkError, 'Could not add',
self.interface.configure_tenant_networks, task)
client_mock.assert_called_once_with(context=task.context)
def test__bind_flat_ports_set_binding_host_id_raise(self, update_mock):
update_mock.side_effect = (neutron_exceptions.ConnectionFailed())
extra = {'vif_port_id': 'foo'}
utils.create_test_port(self.context, node_id=self.node.id,
address='52:54:00:cf:2d:33', extra=extra,
uuid=uuidutils.generate_uuid())
with task_manager.acquire(self.context, self.node.id) as task:
self.assertRaises(exception.NetworkError,
self.interface._bind_flat_ports, task)
def _cs_request(self, *args, **kwargs):
kargs = {}
kargs.setdefault('headers', kwargs.get('headers', {}))
kargs['headers']['User-Agent'] = self.USER_AGENT
if 'content_type' in kwargs:
kargs['headers']['Content-Type'] = kwargs['content_type']
kargs['headers']['Accept'] = kwargs['content_type']
else:
kargs['headers']['Content-Type'] = self.content_type
kargs['headers']['Accept'] = self.content_type
if 'body' in kwargs:
kargs['body'] = kwargs['body']
args = utils.safe_encode_list(args)
kargs = utils.safe_encode_dict(kargs)
if self.log_credentials:
log_kargs = kargs
else:
log_kargs = self._strip_credentials(kargs)
utils.http_log_req(_logger, args, log_kargs)
try:
resp, body = self.request(*args, **kargs)
except httplib2.SSLHandshakeError as e:
if not _logger.isEnabledFor(logging.DEBUG):
utils.http_log_req(_logger,
args,
log_kargs,
force_logging=True)
raise exceptions.SslCertificateValidationError(reason=e)
except Exception as e:
if not _logger.isEnabledFor(logging.DEBUG):
utils.http_log_req(_logger,
args,
log_kargs,
force_logging=True)
# Wrap the low-level connection error (socket timeout, redirect
# limit, decompression error, etc) into our custom high-level
# connection exception (it is excepted in the upper layers of code)
_logger.debug("throwing ConnectionFailed : %s", e)
raise exceptions.ConnectionFailed(reason=e)
finally:
# Temporary Fix for gate failures. RPC calls and HTTP requests
# seem to be stepping on each other resulting in bogus fd's being
# picked up for making http requests
self.connections.clear()
utils.http_log_resp(_logger, resp, body)
status_code = self.get_status_code(resp)
if status_code == 401:
raise exceptions.Unauthorized(message=body)
elif status_code == 403:
raise exceptions.Forbidden(message=body)
return resp, body
def test_add_provisioning_network_binding_host_id_raise(self, client_mock):
client_mock.return_value.update_port.side_effect = \
(neutron_exceptions.ConnectionFailed())
extra = {'vif_port_id': 'foo'}
utils.create_test_port(self.context,
node_id=self.node.id,
address='52:54:00:cf:2d:33',
extra=extra,
uuid=uuidutils.generate_uuid())
with task_manager.acquire(self.context, self.node.id) as task:
self.assertRaises(exception.NetworkError,
self.interface.add_provisioning_network, task)
def test_unshelve_offloaded_fails_due_to_neutron(self):
server = self._create_server(networks=[{
'port':
self.neutron.port_1['id']
}],
az='nova:host1')
# with default config shelve means immediate offload as well
req = {'shelve': {}}
self.api.post_server_action(server['id'], req)
self._wait_for_server_parameter(server, {
'status': 'SHELVED_OFFLOADED',
'OS-EXT-SRV-ATTR:host': None
})
allocations = self.placement_api.get('/allocations/%s' %
server['id']).body['allocations']
self.assertEqual(0, len(allocations))
# disable the original host of the instance to force a port update
# during unshelve
source_service_id = self.api.get_services(
host='host1', binary='nova-compute')[0]['id']
self.api.put_service(source_service_id, {"status": "disabled"})
# Simulate that port update fails during unshelve due to neutron is
# unavailable
with mock.patch('nova.tests.fixtures.NeutronFixture.'
'update_port') as mock_update_port:
mock_update_port.side_effect = neutron_exception.ConnectionFailed(
reason='test')
req = {'unshelve': None}
self.api.post_server_action(server['id'], req)
fake_notifier.wait_for_versioned_notifications(
'instance.unshelve.start')
self._wait_for_server_parameter(
server, {
'status': 'SHELVED_OFFLOADED',
'OS-EXT-STS:task_state': None,
'OS-EXT-SRV-ATTR:host': None
})
# As the instance went back to offloaded state we expect no allocation
allocations = self.placement_api.get('/allocations/%s' %
server['id']).body['allocations']
self.assertEqual(0, len(allocations))
def _cs_request(self, *args, **kwargs):
kargs = {}
kargs.setdefault('headers', kwargs.get('headers', {}))
kargs['headers']['User-Agent'] = USER_AGENT
if 'body' in kwargs:
kargs['body'] = kwargs['body']
if self.log_credentials:
log_kargs = kargs
else:
log_kargs = self._strip_credentials(kargs)
utils.http_log_req(_logger, args, log_kargs)
try:
resp, body = self.request(*args, **kargs)
except requests.exceptions.SSLError as e:
raise exceptions.SslCertificateValidationError(reason=str(e))
except Exception as e:
# Wrap the low-level connection error (socket timeout, redirect
# limit, decompression error, etc) into our custom high-level
# connection exception (it is excepted in the upper layers of code)
_logger.debug("throwing ConnectionFailed : %s", e)
raise exceptions.ConnectionFailed(reason=str(e))
utils.http_log_resp(_logger, resp, body)
# log request-id for each api call
request_id = resp.headers.get('x-openstack-request-id')
if request_id:
_logger.debug(
'%(method)s call to neutron for '
'%(url)s used request id '
'%(response_request_id)s', {
'method': resp.request.method,
'url': resp.url,
'response_request_id': request_id
})
if resp.status_code == 401:
raise exceptions.Unauthorized(message=body)
return resp, body
def fake_list_resource(t_ctx, q_ctx, pod, body, _type):
raise q_exceptions.ConnectionFailed()
