def request(self, method, amp, path='/', **kwargs):
LOG.debug("request url %s", path)
_request = getattr(self.session, method.lower())
_url = self._base_url(amp.lb_network_ip) + path
LOG.debug("request url " + _url)
timeout_tuple = (CONF.haproxy_amphora.rest_request_conn_timeout,
CONF.haproxy_amphora.rest_request_read_timeout)
reqargs = {
'url': _url,
'timeout': timeout_tuple,
}
reqargs.update(kwargs)
headers = reqargs.setdefault('headers', {})
headers['User-Agent'] = OCTAVIA_API_CLIENT
# Keep retrying
for a in six.moves.xrange(CONF.haproxy_amphora.connection_max_retries):
try:
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
message="A true SSLContext object is not available")
r = _request(**reqargs)
LOG.debug(
"Connected to amphora. Response: {resp}".format(resp=r))
return r
except (requests.ConnectionError, requests.Timeout):
LOG.warning(_LW("Could not connect to instance. Retrying."))
time.sleep(CONF.haproxy_amphora.connection_retry_interval)
LOG.error(
_LE("Connection retries (currently set to %s) "
"exhausted. The amphora is unavailable."),
CONF.haproxy_amphora.connection_max_retries)
raise driver_except.TimeOutException()
def request(self, method, amp, path='/', **kwargs):
LOG.debug("request url %s", path)
_request = getattr(self.session, method.lower())
_url = self._base_url(amp.lb_network_ip) + path
LOG.debug("request url " + _url)
timeout_tuple = (CONF.haproxy_amphora.rest_request_conn_timeout,
CONF.haproxy_amphora.rest_request_read_timeout)
reqargs = {
'verify': CONF.haproxy_amphora.server_ca,
'url': _url,
'timeout': timeout_tuple,
}
reqargs.update(kwargs)
headers = reqargs.setdefault('headers', {})
headers['User-Agent'] = OCTAVIA_API_CLIENT
self.ssl_adapter.uuid = amp.id
# Keep retrying
for a in six.moves.xrange(CONF.haproxy_amphora.connection_max_retries):
try:
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
message="A true SSLContext object is not available")
r = _request(**reqargs)
except (requests.ConnectionError, requests.Timeout):
LOG.warn(_LW("Could not connect to instance. Retrying."))
time.sleep(CONF.haproxy_amphora.connection_retry_interval)
if a >= CONF.haproxy_amphora.connection_max_retries:
raise driver_except.TimeOutException()
else:
return r
raise driver_except.UnavailableException()
def request(self, method, amp, path='/', **kwargs):
LOG.debug("request url %s", path)
_request = getattr(self.session, method.lower())
_url = self._base_url(amp.lb_network_ip) + path
LOG.debug("request url %s", _url)
timeout_tuple = (CONF.haproxy_amphora.rest_request_conn_timeout,
CONF.haproxy_amphora.rest_request_read_timeout)
reqargs = {
'verify': CONF.haproxy_amphora.server_ca,
'url': _url,
'timeout': timeout_tuple, }
reqargs.update(kwargs)
headers = reqargs.setdefault('headers', {})
headers['User-Agent'] = OCTAVIA_API_CLIENT
self.ssl_adapter.uuid = amp.id
exception = None
# Keep retrying
for a in six.moves.xrange(CONF.haproxy_amphora.connection_max_retries):
try:
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
message="A true SSLContext object is not available"
)
r = _request(**reqargs)
LOG.debug('Connected to amphora. Response: %(resp)s',
{'resp': r})
content_type = r.headers.get('content-type', '')
# Check the 404 to see if it is just that the network in the
# amphora is not yet up, in which case retry.
# Otherwise return the response quickly.
if r.status_code == 404:
LOG.debug('Got a 404 (content-type: %(content_type)s) -- '
'connection data: %(content)s',
{'content_type': content_type,
'content': r.content})
if content_type.find("application/json") == -1:
LOG.debug("Amphora agent not ready.")
raise requests.ConnectionError
try:
json_data = r.json().get('details', '')
if 'No suitable network interface found' in json_data:
LOG.debug("Amphora network interface not found.")
raise requests.ConnectionError
except simplejson.JSONDecodeError: # if r.json() fails
pass # TODO(rm_work) Should we do something?
return r
except (requests.ConnectionError, requests.Timeout) as e:
exception = e
LOG.warning("Could not connect to instance. Retrying.")
time.sleep(CONF.haproxy_amphora.connection_retry_interval)
LOG.error("Connection retries (currently set to %(max_retries)s) "
"exhausted. The amphora is unavailable. Reason: "
"%(exception)s",
{'max_retries': CONF.haproxy_amphora.connection_max_retries,
'exception': exception})
raise driver_except.TimeOutException()
def request(self, method, amp, path='/', **kwargs):
LOG.debug("request url %s", path)
_request = getattr(self.session, method.lower())
_url = self._base_url(amp.lb_network_ip) + path
reqargs = {
'verify': CONF.haproxy_amphora.server_ca,
'url': _url,
}
reqargs.update(kwargs)
headers = reqargs.setdefault('headers', {})
headers['User-Agent'] = OCTAVIA_API_CLIENT
self.ssl_adapter.uuid = amp.id
# Keep retrying
for a in six.moves.xrange(CONF.haproxy_amphora.connection_max_retries):
try:
r = _request(**reqargs)
except requests.ConnectionError:
LOG.warn(_LW("Could not talk to instance"))
time.sleep(CONF.haproxy_amphora.connection_retry_interval)
if a >= CONF.haproxy_amphora.connection_max_retries:
raise driver_except.TimeOutException()
else:
return r
raise driver_except.UnavailableException()
def _connect(self, hostname):
for attempts in six.moves.xrange(
self.amp_config.connection_max_retries):
try:
self.client.connect(hostname=hostname,
username=self.amp_config.username,
key_filename=self.amp_config.key_path)
except socket.error:
LOG.warn(_LW("Could not ssh to instance"))
time.sleep(self.amp_config.connection_retry_interval)
if attempts >= self.amp_config.connection_max_retries:
raise exc.TimeOutException()
else:
return
raise exc.UnavailableException()
def test_amphora_compute_connectivity_wait(self, mock_driver,
mock_generate_uuid, mock_log,
mock_get_session,
mock_listener_repo_get,
mock_listener_repo_update,
mock_amphora_repo_update):
amp_compute_conn_wait_obj = (
amphora_driver_tasks.AmphoraComputeConnectivityWait())
amp_compute_conn_wait_obj.execute(_amphora_mock)
mock_driver.get_info.assert_called_once_with(_amphora_mock)
mock_driver.get_info.side_effect = driver_except.TimeOutException()
self.assertRaises(driver_except.TimeOutException,
amp_compute_conn_wait_obj.execute, _amphora_mock)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, AMP_ID, status=constants.ERROR)
def request(self, method, amp, path='/', **kwargs):
LOG.debug("request url %s", path)
_request = getattr(self.session, method.lower())
_url = self._base_url(amp.lb_network_ip) + path
LOG.debug("request url " + _url)
timeout_tuple = (CONF.haproxy_amphora.rest_request_conn_timeout,
CONF.haproxy_amphora.rest_request_read_timeout)
reqargs = {
'verify': CONF.haproxy_amphora.server_ca,
'url': _url,
'timeout': timeout_tuple, }
reqargs.update(kwargs)
headers = reqargs.setdefault('headers', {})
headers['User-Agent'] = OCTAVIA_API_CLIENT
self.ssl_adapter.uuid = amp.id
retry_attempt = False
exception = None
# Keep retrying
for a in six.moves.xrange(CONF.haproxy_amphora.connection_max_retries):
try:
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
message="A true SSLContext object is not available"
)
r = _request(**reqargs)
LOG.debug("Connected to amphora. Response: {resp}".format(
resp=r))
# Give a 404 response one retry. Flask/werkzeug is
# returning 404 on startup.
if r.status_code == 404 and retry_attempt is False:
retry_attempt = True
raise requests.ConnectionError
return r
except (requests.ConnectionError, requests.Timeout) as e:
exception = e
LOG.warning(_LW("Could not connect to instance. Retrying."))
time.sleep(CONF.haproxy_amphora.connection_retry_interval)
LOG.error(_LE("Connection retries (currently set to %(max_retries)s) "
"exhausted. The amphora is unavailable. Reason: "
"%(exception)s"),
{'max_retries': CONF.haproxy_amphora.connection_max_retries,
'exception': exception})
raise driver_except.TimeOutException()
def request(self,
method,
amp,
path='/',
timeout_dict=None,
retry_404=True,
raise_retry_exception=False,
**kwargs):
cfg_ha_amp = CONF.haproxy_amphora
if timeout_dict is None:
timeout_dict = {}
req_conn_timeout = timeout_dict.get(
consts.REQ_CONN_TIMEOUT, cfg_ha_amp.rest_request_conn_timeout)
req_read_timeout = timeout_dict.get(
consts.REQ_READ_TIMEOUT, cfg_ha_amp.rest_request_read_timeout)
conn_max_retries = timeout_dict.get(consts.CONN_MAX_RETRIES,
cfg_ha_amp.connection_max_retries)
conn_retry_interval = timeout_dict.get(
consts.CONN_RETRY_INTERVAL, cfg_ha_amp.connection_retry_interval)
LOG.debug("request url %s", path)
_request = getattr(self.session, method.lower())
_url = self._base_url(amp.lb_network_ip, amp.api_version) + path
LOG.debug("request url %s", _url)
reqargs = {
'verify': CONF.haproxy_amphora.server_ca,
'url': _url,
'timeout': (req_conn_timeout, req_read_timeout),
}
reqargs.update(kwargs)
headers = reqargs.setdefault('headers', {})
headers['User-Agent'] = OCTAVIA_API_CLIENT
self.ssl_adapter.uuid = amp.id
exception = None
# Keep retrying
for dummy in range(conn_max_retries):
try:
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
message="A true SSLContext object is not available")
r = _request(**reqargs)
LOG.debug('Connected to amphora. Response: %(resp)s',
{'resp': r})
content_type = r.headers.get('content-type', '')
# Check the 404 to see if it is just that the network in the
# amphora is not yet up, in which case retry.
# Otherwise return the response quickly.
if r.status_code == 404:
if not retry_404:
raise exc.NotFound()
LOG.debug(
'Got a 404 (content-type: %(content_type)s) -- '
'connection data: %(content)s', {
'content_type': content_type,
'content': r.content
})
if content_type.find("application/json") == -1:
LOG.debug("Amphora agent not ready.")
raise requests.ConnectionError
try:
json_data = r.json().get('details', '')
if 'No suitable network interface found' in json_data:
LOG.debug("Amphora network interface not found.")
raise requests.ConnectionError
except simplejson.JSONDecodeError: # if r.json() fails
pass # TODO(rm_work) Should we do something?
return r
except (requests.ConnectionError, requests.Timeout) as e:
exception = e
LOG.warning("Could not connect to instance. Retrying.")
time.sleep(conn_retry_interval)
if raise_retry_exception:
# For taskflow persistence cause attribute should
# be serializable to JSON. Pass None, as cause exception
# is described in the expection message.
raise driver_except.AmpConnectionRetry(
exception=str(e)) from None
LOG.error(
"Connection retries (currently set to %(max_retries)s) "
"exhausted. The amphora is unavailable. Reason: "
"%(exception)s", {
'max_retries': conn_max_retries,
'exception': exception
})
raise driver_except.TimeOutException()
