def api_providers(self):
"""Parse api_providers from response.
Returns: api_providers in [(host, port, is_ssl), ...] format
"""
def _provider_from_listen_addr(addr):
# (pssl|ptcp):<ip>:<port> => (host, port, is_ssl)
parts = addr.split(':')
return (parts[1], int(parts[2]), parts[0] == 'pssl')
try:
if self.successful():
ret = []
body = jsonutils.loads(self.value.body)
for node in body.get('results', []):
for role in node.get('roles', []):
if role.get('role') == 'api_provider':
addr = role.get('listen_addr')
if addr:
ret.append(_provider_from_listen_addr(addr))
return ret
except Exception as e:
LOG.warn(_LW("[%(rid)d] Failed to parse API provider: %(e)s"), {
'rid': self._rid(),
'e': e
})
# intentionally fall through
return None
def _issue_request(self):
'''Issue a request to a provider.'''
conn = self.get_conn()
if conn is None:
error = Exception("No API connections available")
self._request_error = error
return error
url = self._url
LOG.debug(
"[%(rid)d] Issuing - request url: %(conn)s, body: %(body)s", {
'rid': self._rid(),
'conn': self._request_str(conn, url),
'body': self._body
})
issued_time = time.time()
is_conn_error = False
is_conn_service_unavail = False
response = None
try:
redirects = 0
while redirects <= self._redirects:
# Update connection with user specified request timeout,
# the connect timeout is usually smaller so we only set
# the request timeout after a connection is established
if conn.sock is None:
conn.connect()
conn.sock.settimeout(self._http_timeout)
elif conn.sock.gettimeout() != self._http_timeout:
conn.sock.settimeout(self._http_timeout)
headers = copy.copy(self._headers)
if templates.RELOGIN in url:
url = jsonutils.loads(templates.LOGIN)['path']
conn.connect()
self._api_client._wait_for_login(conn, headers)
url = self._url
cookie = self._api_client.auth_cookie(conn)
if (self._url != jsonutils.loads(templates.LOGIN)['path']
and cookie):
headers['Cookie'] = cookie['Cookie']
headers['X-CSRFTOKEN'] = cookie['X-CSRFTOKEN']
try:
if self._body:
body = jsonutils.dumps(self._body)
else:
body = None
LOG.debug(
"Issuing request: self._method = [%(method)s], "
"url= %(url)s, body=%(body)s, "
"headers=%(headers)s", {
'method': self._method,
"url": url,
"body": body,
"headers": headers
})
conn.request(self._method, url, body, headers)
except Exception as e:
LOG.warning(
_LW("[%(rid)d] Exception issuing request: %(e)s"), {
'rid': self._rid(),
'e': e
})
raise e
response = conn.getresponse()
response.body = response.read()
response.headers = response.getheaders()
elapsed_time = time.time() - issued_time
LOG.debug(
"@@@@@@ [ _issue_request ] [%(rid)d] "
"Completed request '%(conn)s': "
"%(status)s (%(elapsed)s seconds), "
"response.headers %(response.headers)s, "
"response.body %(response.body)s", {
'rid': self._rid(),
'conn': self._request_str(conn, url),
'status': response.status,
'elapsed': elapsed_time,
'response.headers': response.headers,
'response.body': response.body
})
if response.status in (401, 302):
if (cookie is None and self._url != jsonutils.loads(
templates.LOGIN)['path']):
# The connection still has no valid cookie despite
# attempts to authenticate and the request has failed
# with unauthorized status code. If this isn't a
# a request to authenticate, we should abort the
# request since there is no point in retrying.
self._abort = True
# If request is unauthorized, clear the session cookie
# for the current provider so that subsequent requests
# to the same provider triggers re-authentication.
self._api_client.set_auth_cookie(conn, None)
elif 503 == response.status:
is_conn_service_unavail = True
if response.status not in [301, 307]:
break
elif redirects >= self._redirects:
LOG.info(
_LI("[%d] Maximum redirects exceeded, aborting "
"request"), self._rid())
break
redirects += 1
conn, url = self._redirect_params(conn, response.headers,
self._client_conn is None)
if url is None:
response.status = 500
break
LOG.info(_LI("[%(rid)d] Redirecting request to: %(conn)s"), {
'rid': self._rid(),
'conn': self._request_str(conn, url)
})
# yield here, just in case we are not out of the loop yet
eventlet.greenthread.sleep(0)
# If we receive any of these responses, then
# our server did not process our request and may be in an
# errored state. Raise an exception, which will cause the
# the conn to be released with is_conn_error == True
# which puts the conn on the back of the client's priority
# queue.
if 500 == response.status or 501 < response.status:
LOG.warning(
_LW("[%(rid)d] Request '%(method)s %(url)s' "
"received: %(status)s"), {
'rid': self._rid(),
'method': self._method,
'url': self._url,
'status': response.status
})
raise Exception('Server error return: %s', response.status)
return response
except Exception as e:
if isinstance(e, httpclient.BadStatusLine):
msg = ("Invalid server response")
else:
msg = str(e)
if response is None:
elapsed_time = time.time() - issued_time
LOG.warn(
_LW("[%(rid)d] Failed request '%(conn)s': '%(msg)s' "
"(%(elapsed)s seconds)"), {
'rid': self._rid(),
'conn': self._request_str(conn, url),
'msg': msg,
'elapsed': elapsed_time
})
self._request_error = e
is_conn_error = True
return e
finally:
# Make sure we release the original connection provided by the
# acquire_connection() call above.
if self._client_conn is None:
self._api_client.release_connection(conn,
is_conn_error,
is_conn_service_unavail,
rid=self._rid())
def _redirect_params(self, conn, headers, allow_release_conn=False):
"""Process redirect response, create new connection if necessary.
Args:
conn: connection that returned the redirect response
headers: response headers of the redirect response
allow_release_conn: if redirecting to a different server,
release existing connection back to connection pool.
Returns: Return tuple(conn, url) where conn is a connection object
to the redirect target and url is the path of the API request
"""
url = None
for name, value in headers:
if name.lower() == "location":
url = value
break
if not url:
LOG.warn(
_LW("[%d] Received redirect status without location "
"header field"), self._rid())
return (conn, None)
# Accept location with the following format:
# 1. /path, redirect to same node
# 2. scheme://hostname:[port]/path where scheme is https or http
# Reject others
# 3. e.g. relative paths, unsupported scheme, unspecified host
result = urlparse.urlparse(url)
if not result.scheme and not result.hostname and result.path:
if result.path[0] == "/":
if result.query:
url = "%s?%s" % (result.path, result.query)
else:
url = result.path
return (conn, url) # case 1
else:
LOG.warn(
_LW("[%(rid)d] Received invalid redirect location: "
"'%(url)s'"), {
'rid': self._rid(),
'url': url
})
return (conn, None) # case 3
elif result.scheme not in ["http", "https"] or not result.hostname:
LOG.warn(
_LW("[%(rid)d] Received malformed redirect "
"location: %(url)s"), {
'rid': self._rid(),
'url': url
})
return (conn, None) # case 3
# case 2, redirect location includes a scheme
# so setup a new connection and authenticate
if allow_release_conn:
self._api_client.release_connection(conn)
conn_params = (result.hostname, result.port, result.scheme == "https")
conn = self._api_client.acquire_redirect_connection(
conn_params, True, self._headers)
if result.query:
url = "%s?%s" % (result.path, result.query)
else:
url = result.path
return (conn, url)
def request(self, opt, content_type="application/json", **message):
'''Issues request to controller.'''
self.message = self._render(getattr(templates, opt), **message)
method = self.message['method']
url = self.message['path']
body = self.message['body'] if 'body' in self.message else None
g = eventlet_request.GenericRequestEventlet(
self,
method,
url,
body,
content_type,
auto_login=True,
http_timeout=self._http_timeout,
retries=self._retries,
redirects=self._redirects)
g.start()
response = g.join()
# response is a modified HTTPResponse object or None.
# response.read() will not work on response as the underlying library
# request_eventlet.ApiRequestEventlet has already called this
# method in order to extract the body and headers for processing.
# ApiRequestEventlet derived classes call .read() and
# .getheaders() on the HTTPResponse objects and store the results in
# the response object's .body and .headers data members for future
# access.
if response is None:
# Timeout.
LOG.error(_LE('Request timed out: %(method)s to %(url)s'), {
'method': method,
'url': url
})
raise exception.RequestTimeout()
status = response.status
if status == 401:
raise exception.UnAuthorizedRequest()
# Fail-fast: Check for exception conditions and raise the
# appropriate exceptions for known error codes.
if status in [404]:
LOG.warning(
_LW("Resource not found. Response status: %(status)s, "
"response body: %(response.body)s"), {
'status': status,
'response.body': response.body
})
exception.ERROR_MAPPINGS[status](response)
elif status in exception.ERROR_MAPPINGS:
LOG.error(_LE("Received error code: %s"), status)
LOG.error(_LE("Server Error Message: %s"), response.body)
exception.ERROR_MAPPINGS[status](response)
# Continue processing for non-error condition.
if (status != 200 and status != 201 and status != 204):
LOG.error(
_LE("%(method)s to %(url)s, unexpected response code: "
"%(status)d (content = '%(body)s')"), {
'method': method,
'url': url,
'status': response.status,
'body': response.body
})
return None
if url == jsonutils.loads(templates.LOGOUT)['path']:
return response.body
else:
try:
return jsonutils.loads(response.body)
except UnicodeDecodeError:
LOG.debug(
"The following strings cannot be decoded with "
"'utf-8, trying 'ISO-8859-1' instead. %(body)s",
{'body': response.body})
return jsonutils.loads(response.body, encoding='ISO-8859-1')
except Exception as e:
LOG.error(_LE("Decode error, the response.body %(body)s"),
{'body': response.body})
raise e
def release_connection(self, http_conn, bad_state=False,
service_unavail=False, rid=-1):
'''Mark HTTPConnection instance as available for check-out.
:param http_conn: An HTTPConnection instance obtained from this
instance.
:param bad_state: True if http_conn is known to be in a bad state
(e.g. connection fault.)
:service_unavail: True if http_conn returned 503 response.
:param rid: request id passed in from request eventlet.
'''
conn_params = self._conn_params(http_conn)
if self._conn_params(http_conn) not in self._api_providers:
LOG.debug("[%(rid)d] Released connection %(conn)s is not an "
"API provider for the cluster",
{'rid': rid,
'conn': api_client.ctrl_conn_to_str(http_conn)})
return
elif hasattr(http_conn, "no_release"):
return
if bad_state:
# Reconnect to provider.
LOG.warning(_LW("[%(rid)d] Connection returned in bad state, "
"reconnecting to %(conn)s"),
{'rid': rid,
'conn': api_client.ctrl_conn_to_str(http_conn)})
http_conn.close()
http_conn = self._create_connection(*self._conn_params(http_conn))
conns = []
while not self._conn_pool.empty():
priority, conn = self._conn_pool.get()
if self._conn_params(conn) == conn_params:
conn.close()
continue
conns.append((priority, conn))
for priority, conn in conns:
self._conn_pool.put((priority, conn))
priority = self._next_conn_priority
self._next_conn_priority += 1
elif service_unavail:
# http_conn returned a service unaviable response, put other
# connections to the same controller at end of priority queue,
conns = []
while not self._conn_pool.empty():
priority, conn = self._conn_pool.get()
if self._conn_params(conn) == conn_params:
priority = self._next_conn_priority
self._next_conn_priority += 1
conns.append((priority, conn))
for priority, conn in conns:
self._conn_pool.put((priority, conn))
priority = self._next_conn_priority
self._next_conn_priority += 1
else:
priority = http_conn.priority
self._conn_pool.put((priority, http_conn))
LOG.debug("[%(rid)d] Released connection %(conn)s. %(qsize)d "
"connection(s) available.",
{'rid': rid, 'conn': api_client.ctrl_conn_to_str(http_conn),
'qsize': self._conn_pool.qsize()})