def _check_keystone_extensions(self, url):
"""Calls Keystone URL and detects the available extensions."""
try:
if not url.endswith("/"):
url += '/'
resp, body = self.request("%sextensions" % url,
"GET",
headers={'Accept': 'application/json'})
if resp.status_code in (200, 204): # some cases we get No Content
if 'extensions' in body and 'values' in body['extensions']:
# Parse correct format (per contract)
extensions = body['extensions']['values']
elif 'extensions' in body:
# Support incorrect, but prevalent format
extensions = body['extensions']
else:
return dict(
message=('Unrecognized extensions response from %s' %
url))
return dict(self._get_extension_info(e) for e in extensions)
elif resp.status_code == 305:
return self._check_keystone_extensions(resp['location'])
else:
raise exceptions.from_response(resp, "GET",
"%sextensions" % url)
except Exception as e:
_logger.exception(e)
def _check_keystone_extensions(self, url):
"""Call Keystone URL and detects the available extensions."""
try:
if not url.endswith("/"):
url += '/'
resp, body = self._request("%sextensions" % url, "GET",
headers={'Accept':
'application/json'})
if resp.status_code in (200, 204): # some cases we get No Content
if 'extensions' in body and 'values' in body['extensions']:
# Parse correct format (per contract)
extensions = body['extensions']['values']
elif 'extensions' in body:
# Support incorrect, but prevalent format
extensions = body['extensions']
else:
return dict(message=(
_('Unrecognized extensions response from %s') % url))
return dict(self._get_extension_info(e) for e in extensions)
elif resp.status_code == 305:
return self._check_keystone_extensions(resp['location'])
else:
raise exceptions.from_response(
resp, "GET", "%sextensions" % url)
except Exception:
_logger.exception('Failed to check keystone extensions.')
def request(self, url, method, **kwargs):
"""Send an http request with the specified characteristics.
Wrapper around requests.request to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
"""
# Copy the kwargs so we can reuse the original in case of redirects
request_kwargs = copy.copy(kwargs)
request_kwargs.setdefault('headers', kwargs.get('headers', {}))
request_kwargs['headers']['User-Agent'] = self.USER_AGENT
if self.original_ip:
request_kwargs['headers']['Forwarded'] = "for=%s;by=%s" % (
self.original_ip, self.USER_AGENT)
if 'body' in kwargs:
request_kwargs['headers']['Content-Type'] = 'application/json'
request_kwargs['data'] = self.serialize(kwargs['body'])
del request_kwargs['body']
if self.cert:
request_kwargs['cert'] = self.cert
if self.timeout is not None:
request_kwargs.setdefault('timeout', self.timeout)
self.http_log_req((
url,
method,
), request_kwargs)
try:
resp = requests.request(method,
url,
verify=self.verify_cert,
**request_kwargs)
except requests.ConnectionError:
msg = 'Unable to establish connection to %s' % url
raise exceptions.ClientException(msg)
self.http_log_resp(resp)
if resp.text:
try:
body = json.loads(resp.text)
except (ValueError, TypeError):
body = None
_logger.debug("Could not decode JSON from body: %s" %
resp.text)
else:
_logger.debug("No body was returned.")
body = None
if resp.status_code >= 400:
_logger.debug("Request returned failure status: %s",
resp.status_code)
raise exceptions.from_response(resp, body or resp.text)
elif resp.status_code in (301, 302, 305):
# Redirected. Reissue the request to the new location.
return self.request(resp.headers['location'], method, **kwargs)
return resp, body
def request(self, url, method, **kwargs):
"""Send an http request with the specified characteristics.
Wrapper around requests.request to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
"""
# Copy the kwargs so we can reuse the original in case of redirects
request_kwargs = copy.copy(kwargs)
request_kwargs.setdefault('headers', kwargs.get('headers', {}))
request_kwargs['headers']['User-Agent'] = self.USER_AGENT
if self.original_ip:
request_kwargs['headers']['Forwarded'] = "for=%s;by=%s" % (
self.original_ip, self.USER_AGENT)
if 'body' in kwargs:
request_kwargs['headers']['Content-Type'] = 'application/json'
request_kwargs['data'] = self.serialize(kwargs['body'])
del request_kwargs['body']
if self.cert:
request_kwargs['cert'] = self.cert
if self.timeout is not None:
request_kwargs.setdefault('timeout', self.timeout)
self.http_log_req((url, method,), request_kwargs)
try:
resp = requests.request(
method,
url,
verify=self.verify_cert,
**request_kwargs)
except requests.ConnectionError:
msg = 'Unable to establish connection to %s' % url
raise exceptions.ClientException(msg)
self.http_log_resp(resp)
if resp.text:
try:
body = json.loads(resp.text)
except (ValueError, TypeError):
body = None
_logger.debug("Could not decode JSON from body: %s"
% resp.text)
else:
_logger.debug("No body was returned.")
body = None
if resp.status_code >= 400:
_logger.debug(
"Request returned failure status: %s",
resp.status_code)
raise exceptions.from_response(resp, body or resp.text)
elif resp.status_code in (301, 302, 305):
# Redirected. Reissue the request to the new location.
return self.request(resp.headers['location'], method, **kwargs)
return resp, body
def assert_exception(self, ex_cls, method, url, status_code, json_data):
ex = exceptions.from_response(
FakeResponse(status_code=status_code,
headers={"Content-Type": "application/json"},
json_data=json_data), method, url)
self.assertIsInstance(ex, ex_cls)
self.assertEqual(ex.message, json_data["error"]["message"])
self.assertEqual(ex.details, json_data["error"]["details"])
self.assertEqual(ex.method, method)
self.assertEqual(ex.url, url)
self.assertEqual(ex.http_status, status_code)
def assert_exception(self, ex_cls, method, url, status_code, json_data):
ex = exceptions.from_response(
FakeResponse(status_code=status_code,
headers={"Content-Type": "application/json"},
json_data=json_data),
method,
url)
self.assertIsInstance(ex, ex_cls)
self.assertEqual(ex.message, json_data["error"]["message"])
self.assertEqual(ex.details, json_data["error"]["details"])
self.assertEqual(ex.method, method)
self.assertEqual(ex.url, url)
self.assertEqual(ex.http_status, status_code)
def _check_keystone_versions(self, url):
"""Calls Keystone URL and detects the available API versions."""
try:
client = httpclient.HTTPClient()
resp, body = client.request(url,
"GET",
headers={'Accept': 'application/json'})
# Multiple Choices status code is returned by the root
# identity endpoint, with references to one or more
# Identity API versions -- v3 spec
# some cases we get No Content
if resp.status_code in (200, 204, 300):
try:
results = {}
if 'version' in body:
results['message'] = "Keystone found at %s" % url
version = body['version']
# Stable/diablo incorrect format
id, status, version_url = \
self._get_version_info(version, url)
results[str(id)] = {
"id": id,
"status": status,
"url": version_url
}
return results
elif 'versions' in body:
# Correct format
results['message'] = "Keystone found at %s" % url
for version in body['versions']['values']:
id, status, version_url = \
self._get_version_info(version, url)
results[str(id)] = {
"id": id,
"status": status,
"url": version_url
}
return results
else:
results['message'] = ("Unrecognized response from %s" %
url)
return results
except KeyError:
raise exceptions.AuthorizationFailure()
elif resp.status_code == 305:
return self._check_keystone_versions(resp['location'])
else:
raise exceptions.from_response(resp, "GET", url)
except Exception as e:
_logger.exception(e)
def _check_keystone_versions(self, url):
"""Calls Keystone URL and detects the available API versions."""
try:
httpclient = client.HTTPClient()
resp, body = httpclient.request(url, "GET",
headers={'Accept':
'application/json'})
# Multiple Choices status code is returned by the root
# identity endpoint, with references to one or more
# Identity API versions -- v3 spec
# some cases we get No Content
if resp.status_code in (200, 204, 300):
try:
results = {}
if 'version' in body:
results['message'] = "Keystone found at %s" % url
version = body['version']
# Stable/diablo incorrect format
id, status, version_url = \
self._get_version_info(version, url)
results[str(id)] = {"id": id,
"status": status,
"url": version_url}
return results
elif 'versions' in body:
# Correct format
results['message'] = "Keystone found at %s" % url
for version in body['versions']['values']:
id, status, version_url = \
self._get_version_info(version, url)
results[str(id)] = {"id": id,
"status": status,
"url": version_url}
return results
else:
results['message'] = ("Unrecognized response from %s"
% url)
return results
except KeyError:
raise exceptions.AuthorizationFailure()
elif resp.status_code == 305:
return self._check_keystone_versions(resp['location'])
else:
raise exceptions.from_response(resp, resp.text)
except Exception as e:
_logger.exception(e)
def _check_keystone_versions(self, url):
""" Calls Keystone URL and detects the available API versions """
try:
httpclient = client.HTTPClient()
resp, body = httpclient.request(
url, "GET", headers={'Accept': 'application/json'})
if resp.status in (200, 204): # in some cases we get No Content
try:
results = {}
if 'version' in body:
results['message'] = "Keystone found at %s" % url
version = body['version']
# Stable/diablo incorrect format
id, status, version_url = self._get_version_info(
version, url)
results[str(id)] = {
"id": id,
"status": status,
"url": version_url
}
return results
elif 'versions' in body:
# Correct format
results['message'] = "Keystone found at %s" % url
for version in body['versions']['values']:
id, status, version_url = self._get_version_info(
version, url)
results[str(id)] = {
"id": id,
"status": status,
"url": version_url
}
return results
else:
results['message'] = ("Unrecognized response from %s" %
url)
return results
except KeyError:
raise exceptions.AuthorizationFailure()
elif resp.status == 305:
return self._check_keystone_versions(resp['location'])
else:
raise exceptions.from_response(resp, body)
except Exception as e:
_logger.exception(e)
def request(self, url, method, **kwargs):
""" Send an http request with the specified characteristics.
Wrapper around requests.request to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
"""
# Copy the kwargs so we can reuse the original in case of redirects
request_kwargs = copy.copy(kwargs)
request_kwargs.setdefault("headers", kwargs.get("headers", {}))
request_kwargs["headers"]["User-Agent"] = self.USER_AGENT
if self.original_ip:
request_kwargs["headers"]["Forwarded"] = "for=%s;by=%s" % (self.original_ip, self.USER_AGENT)
if "body" in kwargs:
request_kwargs["headers"]["Content-Type"] = "application/json"
request_kwargs["data"] = self.serialize(kwargs["body"])
del request_kwargs["body"]
if self.cert:
request_kwargs["cert"] = self.cert
if self.timeout is not None:
request_kwargs.setdefault("timeout", self.timeout)
self.http_log_req((url, method), request_kwargs)
resp = requests.request(method, url, verify=self.verify_cert, **request_kwargs)
self.http_log_resp(resp)
if resp.status_code >= 400:
_logger.debug("Request returned failure status: %s", resp.status_code)
raise exceptions.from_response(resp, resp.text)
elif resp.status_code in (301, 302, 305):
# Redirected. Reissue the request to the new location.
return self.request(resp.headers["location"], method, **kwargs)
if resp.text:
try:
body = json.loads(resp.text)
except ValueError:
body = None
_logger.debug("Could not decode JSON from body: %s" % resp.text)
else:
_logger.debug("No body was returned.")
body = None
return resp, body
def _check_keystone_extensions(self, url):
"""Calls Keystone URL and detects the available extensions."""
try:
client = httpclient.HTTPClient()
if not url.endswith("/"):
url += '/'
resp, body = client.request("%sextensions" % url, "GET",
headers={'Accept':
'application/json'})
if resp.status_code in (200, 204): # some cases we get No Content
try:
results = {}
if 'extensions' in body:
if 'values' in body['extensions']:
# Parse correct format (per contract)
for extension in body['extensions']['values']:
alias, name = \
self._get_extension_info(
extension['extension']
)
results[alias] = name
return results
else:
# Support incorrect, but prevalent format
for extension in body['extensions']:
alias, name = \
self._get_extension_info(extension)
results[alias] = name
return results
else:
results['message'] = ("Unrecognized extensions "
"response from %s" % url)
return results
except KeyError:
raise exceptions.AuthorizationFailure()
elif resp.status_code == 305:
return self._check_keystone_extensions(resp['location'])
else:
raise exceptions.from_response(
resp, "GET", "%sextensions" % url)
except Exception as e:
_logger.exception(e)
def _check_keystone_extensions(self, url):
"""Calls Keystone URL and detects the available extensions."""
try:
client = httpclient.HTTPClient()
if not url.endswith("/"):
url += '/'
resp, body = client.request("%sextensions" % url,
"GET",
headers={'Accept': 'application/json'})
if resp.status_code in (200, 204): # some cases we get No Content
try:
results = {}
if 'extensions' in body:
if 'values' in body['extensions']:
# Parse correct format (per contract)
for extension in body['extensions']['values']:
alias, name = \
self._get_extension_info(
extension['extension']
)
results[alias] = name
return results
else:
# Support incorrect, but prevalent format
for extension in body['extensions']:
alias, name = \
self._get_extension_info(extension)
results[alias] = name
return results
else:
results['message'] = ("Unrecognized extensions "
"response from %s" % url)
return results
except KeyError:
raise exceptions.AuthorizationFailure()
elif resp.status_code == 305:
return self._check_keystone_extensions(resp['location'])
else:
raise exceptions.from_response(resp, "GET",
"%sextensions" % url)
except Exception as e:
_logger.exception(e)
def _check_keystone_versions(self, url):
""" Calls Keystone URL and detects the available API versions """
try:
httpclient = client.HTTPClient()
resp, body = httpclient.request(url, "GET",
headers={'Accept':
'application/json'})
if resp.status in (200, 204): # in some cases we get No Content
try:
results = {}
if 'version' in body:
results['message'] = "Keystone found at %s" % url
version = body['version']
# Stable/diablo incorrect format
id, status, version_url = \
self._get_version_info(version, url)
results[str(id)] = {"id": id,
"status": status,
"url": version_url}
return results
elif 'versions' in body:
# Correct format
results['message'] = "Keystone found at %s" % url
for version in body['versions']['values']:
id, status, version_url = \
self._get_version_info(version, url)
results[str(id)] = {"id": id,
"status": status,
"url": version_url}
return results
else:
results['message'] = ("Unrecognized response from %s"
% url)
return results
except KeyError:
raise exceptions.AuthorizationFailure()
elif resp.status == 305:
return self._check_keystone_versions(resp['location'])
else:
raise exceptions.from_response(resp, body)
except Exception as e:
_logger.exception(e)
def request(self, url, method, **kwargs):
""" Send an http request with the specified characteristics.
Wrapper around httplib2.Http.request to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
"""
# Copy the kwargs so we can reuse the original in case of redirects
request_kwargs = copy.copy(kwargs)
request_kwargs.setdefault('headers', kwargs.get('headers', {}))
request_kwargs['headers']['User-Agent'] = self.USER_AGENT
if self.original_ip:
request_kwargs['headers']['Forwarded'] = "for=%s;by=%s" % (
self.original_ip, self.USER_AGENT)
if 'body' in kwargs:
request_kwargs['headers']['Content-Type'] = 'application/json'
request_kwargs['body'] = self.serialize(kwargs['body'])
self.http_log_req((
url,
method,
), request_kwargs)
resp, body = super(HTTPClient, self).request(url, method,
**request_kwargs)
self.http_log_resp(resp, body)
if resp.status in (400, 401, 403, 404, 408, 409, 413, 500, 501):
_logger.debug("Request returned failure status.")
raise exceptions.from_response(resp, body)
elif resp.status in (301, 302, 305):
# Redirected. Reissue the request to the new location.
return self.request(resp['location'], method, **kwargs)
if body:
try:
body = json.loads(body)
except ValueError:
_logger.debug("Could not decode JSON from body: %s" % body)
else:
_logger.debug("No body was returned.")
body = None
return resp, body
def request(self, url, method, **kwargs):
""" Send an http request with the specified characteristics.
Wrapper around httplib2.Http.request to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
"""
# Copy the kwargs so we can reuse the original in case of redirects
request_kwargs = copy.copy(kwargs)
request_kwargs.setdefault('headers', kwargs.get('headers', {}))
request_kwargs['headers']['User-Agent'] = self.USER_AGENT
if self.original_ip:
request_kwargs['headers']['Forwarded'] = "for=%s;by=%s" % (
self.original_ip, self.USER_AGENT)
if 'body' in kwargs:
request_kwargs['headers']['Content-Type'] = 'application/json'
request_kwargs['body'] = json.dumps(kwargs['body'])
self.http_log_req((url, method,), request_kwargs)
resp, body = super(HTTPClient, self).request(url,
method,
**request_kwargs)
self.http_log_resp(resp, body)
if resp.status in (400, 401, 403, 404, 408, 409, 413, 500, 501):
_logger.debug("Request returned failure status.")
raise exceptions.from_response(resp, body)
elif resp.status in (301, 302, 305):
# Redirected. Reissue the request to the new location.
return self.request(resp['location'], method, **kwargs)
if body:
try:
body = json.loads(body)
except ValueError:
_logger.debug("Could not decode JSON from body: %s" % body)
else:
_logger.debug("No body was returned.")
body = None
return resp, body
def _retrieve_data_from_keystone(redis_client, url, tenant, token,
blacklist_ttl, max_cache_life):
"""Retrieve the authentication data from OpenStack Keystone
:param redis_client: redis.Redis object connected to the redis cache
:param url: Keystone Identity URL to authenticate against
:param tenant: tenant id of user data to retrieve
:param token: auth_token for the tenant_id
:param blacklist_ttl: time in milliseconds for blacklisting failed tokens
:param max_cache_life: time in seconds for the maximum time a cache entry
should remain in the cache of valid data
:returns: a keystoneclient.access.AccessInfo on success or None on error
"""
try:
# Try to authenticate the user and get the user information using
# only the data provided, no special administrative tokens required.
# When using the alternative validation method, the service catalog
# identity does not return a service catalog for valid tokens.
if get_conf().alternate_validation is True:
_url = url.rstrip('/') + '/tokens'
validation_url = _url + '/{0}'.format(token)
headers = {
'Accept': 'application/json',
'X-Auth-Token': token
}
resp = requests.get(validation_url, headers=headers)
if resp.status_code >= 400:
LOG.debug('Request returned failure status: {0}'.format(
resp.status_code))
raise exceptions.from_response(resp, 'GET', _url)
try:
resp_data = resp.json()['access']
except (KeyError, ValueError):
raise exceptions.InvalidResponse(response=resp)
access_info = access.AccessInfoV2(**resp_data)
else:
keystone = keystonev2_client.Client(tenant_id=tenant,
token=token,
auth_url=url)
access_info = keystone.get_raw_token_from_identity_service(
auth_url=url, tenant_id=tenant, token=token)
# cache the data so it is easier to access next time
_send_data_to_cache(redis_client, url, access_info, max_cache_life)
return access_info
except (exceptions.AuthorizationFailure, exceptions.Unauthorized) as ex:
# re-raise 413 here and later on respond with 503
if 'HTTP 413' in str(ex):
raise exceptions.RequestEntityTooLarge(
method='POST',
url=url,
http_status=413
)
# Provided data was invalid and authorization failed
msg = 'Failed to authenticate against {0} - {1}'.format(
url,
str(ex)
)
LOG.debug(msg)
# Blacklist the token
_blacklist_token(redis_client, token, blacklist_ttl)
return None
except exceptions.RequestEntityTooLarge:
LOG.debug('Request entity too large error from authentication server.')
raise
except Exception as ex:
# Provided data was invalid or something else went wrong
msg = 'Failed to authenticate against {0} - {1}'.format(
url,
str(ex)
)
LOG.debug(msg)
return None
def request(self, url, method, json=None, original_ip=None,
user_agent=None, redirect=None, **kwargs):
"""Send an HTTP request with the specified characteristics.
Wrapper around `requests.Session.request` to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
Arguments that are not handled are passed through to the requests
library.
:param string url: Fully qualified URL of HTTP request
:param string method: The http method to use. (eg. 'GET', 'POST')
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param dict headers: Headers to be included in the request. (optional)
:param json: Some data to be represented as JSON. (optional)
:param string user_agent: A user_agent to use for the request. If
present will override one present in headers.
(optional)
:param int/bool redirect: the maximum number of redirections that
can be followed by a request. Either an
integer for a specific count or True/False
for forever/never. (optional)
:param kwargs: any other parameter that can be passed to
requests.Session.request (such as `headers`). Except:
'data' will be overwritten by the data in 'json' param.
'allow_redirects' is ignored as redirects are handled
by the session.
:raises exceptions.ClientException: For connection failure, or to
indicate an error response code.
:returns: The response to the request.
"""
headers = kwargs.setdefault('headers', dict())
if self.cert:
kwargs.setdefault('cert', self.cert)
if self.timeout is not None:
kwargs.setdefault('timeout', self.timeout)
if user_agent:
headers['User-Agent'] = user_agent
elif self.user_agent:
user_agent = headers.setdefault('User-Agent', self.user_agent)
else:
user_agent = headers.setdefault('User-Agent', USER_AGENT)
if self.original_ip:
headers.setdefault('Forwarded',
'for=%s;by=%s' % (self.original_ip, user_agent))
if json is not None:
headers['Content-Type'] = 'application/json'
kwargs['data'] = jsonutils.dumps(json)
kwargs.setdefault('verify', self.verify)
string_parts = ['curl -i']
if method:
string_parts.extend(['-X', method])
string_parts.append(url)
if headers:
for header in six.iteritems(headers):
string_parts.append('-H "%s: %s"' % header)
_logger.debug('REQ: %s', ' '.join(string_parts))
data = kwargs.get('data')
if data:
_logger.debug('REQ BODY: %s', data)
# Force disable requests redirect handling. We will manage this below.
kwargs['allow_redirects'] = False
if redirect is None:
redirect = self.redirect
resp = self._send_request(url, method, redirect, **kwargs)
# NOTE(jamielennox): we create a tuple here to be the same as what is
# returned by the requests library.
resp.history = tuple(resp.history)
if resp.status_code >= 400:
_logger.debug('Request returned failure status: %s',
resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp
def request(self,
url,
method,
json=None,
original_ip=None,
user_agent=None,
redirect=None,
authenticated=None,
endpoint_filter=None,
auth=None,
requests_auth=None,
raise_exc=True,
allow_reauth=True,
log=True,
endpoint_override=None,
connect_retries=0,
logger=_logger,
**kwargs):
"""Send an HTTP request with the specified characteristics.
Wrapper around `requests.Session.request` to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
Arguments that are not handled are passed through to the requests
library.
:param string url: Path or fully qualified URL of HTTP request. If only
a path is provided then endpoint_filter must also be
provided such that the base URL can be determined.
If a fully qualified URL is provided then
endpoint_filter will be ignored.
:param string method: The http method to use. (e.g. 'GET', 'POST')
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param dict headers: Headers to be included in the request. (optional)
:param json: Some data to be represented as JSON. (optional)
:param string user_agent: A user_agent to use for the request. If
present will override one present in headers.
(optional)
:param int/bool redirect: the maximum number of redirections that
can be followed by a request. Either an
integer for a specific count or True/False
for forever/never. (optional)
:param int connect_retries: the maximum number of retries that should
be attempted for connection errors.
(optional, defaults to 0 - never retry).
:param bool authenticated: True if a token should be attached to this
request, False if not or None for attach if
an auth_plugin is available.
(optional, defaults to None)
:param dict endpoint_filter: Data to be provided to an auth plugin with
which it should be able to determine an
endpoint to use for this request. If not
provided then URL is expected to be a
fully qualified URL. (optional)
:param str endpoint_override: The URL to use instead of looking up the
endpoint in the auth plugin. This will be
ignored if a fully qualified URL is
provided but take priority over an
endpoint_filter. (optional)
:param auth: The auth plugin to use when authenticating this request.
This will override the plugin that is attached to the
session (if any). (optional)
:type auth: :py:class:`keystoneclient.auth.base.BaseAuthPlugin`
:param requests_auth: A requests library auth plugin that cannot be
passed via kwarg because the `auth` kwarg
collides with our own auth plugins. (optional)
:type requests_auth: :py:class:`requests.auth.AuthBase`
:param bool raise_exc: If True then raise an appropriate exception for
failed HTTP requests. If False then return the
request object. (optional, default True)
:param bool allow_reauth: Allow fetching a new token and retrying the
request on receiving a 401 Unauthorized
response. (optional, default True)
:param bool log: If True then log the request and response data to the
debug log. (optional, default True)
:param logger: The logger object to use to log request and responses.
If not provided the keystoneclient.session default
logger will be used.
:type logger: logging.Logger
:param kwargs: any other parameter that can be passed to
requests.Session.request (such as `headers`). Except:
'data' will be overwritten by the data in 'json' param.
'allow_redirects' is ignored as redirects are handled
by the session.
:raises keystoneclient.exceptions.ClientException: For connection
failure, or to indicate an error response code.
:returns: The response to the request.
"""
headers = kwargs.setdefault('headers', dict())
if authenticated is None:
authenticated = bool(auth or self.auth)
if authenticated:
auth_headers = self.get_auth_headers(auth)
if auth_headers is None:
msg = _('No valid authentication is available')
raise exceptions.AuthorizationFailure(msg)
headers.update(auth_headers)
if osprofiler_web:
headers.update(osprofiler_web.get_trace_id_headers())
# if we are passed a fully qualified URL and an endpoint_filter we
# should ignore the filter. This will make it easier for clients who
# want to overrule the default endpoint_filter data added to all client
# requests. We check fully qualified here by the presence of a host.
if not urllib.parse.urlparse(url).netloc:
base_url = None
if endpoint_override:
base_url = endpoint_override
elif endpoint_filter:
base_url = self.get_endpoint(auth, **endpoint_filter)
if not base_url:
service_type = (endpoint_filter
or {}).get('service_type', 'unknown')
msg = _('Endpoint for %s service') % service_type
raise exceptions.EndpointNotFound(msg)
url = '%s/%s' % (base_url.rstrip('/'), url.lstrip('/'))
if self.cert:
kwargs.setdefault('cert', self.cert)
if self.timeout is not None:
kwargs.setdefault('timeout', self.timeout)
if user_agent:
headers['User-Agent'] = user_agent
elif self.user_agent:
user_agent = headers.setdefault('User-Agent', self.user_agent)
else:
user_agent = headers.setdefault('User-Agent', USER_AGENT)
if self.original_ip:
headers.setdefault('Forwarded',
'for=%s;by=%s' % (self.original_ip, user_agent))
if json is not None:
headers['Content-Type'] = 'application/json'
kwargs['data'] = jsonutils.dumps(json)
kwargs.setdefault('verify', self.verify)
if requests_auth:
kwargs['auth'] = requests_auth
if log:
self._http_log_request(url,
method=method,
data=kwargs.get('data'),
headers=headers,
logger=logger)
# Force disable requests redirect handling. We will manage this below.
kwargs['allow_redirects'] = False
if redirect is None:
redirect = self.redirect
send = functools.partial(self._send_request, url, method, redirect,
log, logger, connect_retries)
try:
connection_params = self.get_auth_connection_params(auth=auth)
except exceptions.MissingAuthPlugin: # nosec(cjschaef)
# NOTE(jamielennox): If we've gotten this far without an auth
# plugin then we should be happy with allowing no additional
# connection params. This will be the typical case for plugins
# anyway.
pass
else:
if connection_params:
kwargs.update(connection_params)
resp = send(**kwargs)
# handle getting a 401 Unauthorized response by invalidating the plugin
# and then retrying the request. This is only tried once.
if resp.status_code == 401 and authenticated and allow_reauth:
if self.invalidate(auth):
auth_headers = self.get_auth_headers(auth)
if auth_headers is not None:
headers.update(auth_headers)
resp = send(**kwargs)
if raise_exc and resp.status_code >= 400:
logger.debug('Request returned failure status: %s',
resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp
def request(self, url, method, json=None, original_ip=None,
user_agent=None, redirect=None, authenticated=None,
endpoint_filter=None, **kwargs):
"""Send an HTTP request with the specified characteristics.
Wrapper around `requests.Session.request` to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
Arguments that are not handled are passed through to the requests
library.
:param string url: Path or fully qualified URL of HTTP request. If only
a path is provided then endpoint_filter must also be
provided such that the base URL can be determined.
If a fully qualified URL is provided then
endpoint_filter will be ignored.
:param string method: The http method to use. (eg. 'GET', 'POST')
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param dict headers: Headers to be included in the request. (optional)
:param json: Some data to be represented as JSON. (optional)
:param string user_agent: A user_agent to use for the request. If
present will override one present in headers.
(optional)
:param int/bool redirect: the maximum number of redirections that
can be followed by a request. Either an
integer for a specific count or True/False
for forever/never. (optional)
:param bool authenticated: True if a token should be attached to this
request, False if not or None for attach if
an auth_plugin is available.
(optional, defaults to None)
:param dict endpoint_filter: Data to be provided to an auth plugin with
which it should be able to determine an
endpoint to use for this request. If not
provided then URL is expected to be a
fully qualified URL. (optional)
:param kwargs: any other parameter that can be passed to
requests.Session.request (such as `headers`). Except:
'data' will be overwritten by the data in 'json' param.
'allow_redirects' is ignored as redirects are handled
by the session.
:raises exceptions.ClientException: For connection failure, or to
indicate an error response code.
:returns: The response to the request.
"""
headers = kwargs.setdefault('headers', dict())
if authenticated is None:
authenticated = self.auth is not None
if authenticated:
token = self.get_token()
if not token:
raise exceptions.AuthorizationFailure("No token Available")
headers['X-Auth-Token'] = token
# if we are passed a fully qualified URL and a endpoint_filter we
# should ignore the filter. This will make it easier for clients who
# want to overrule the default endpoint_filter data added to all client
# requests. We check fully qualified here by the presence of a host.
url_data = urllib.parse.urlparse(url)
if endpoint_filter and not url_data.netloc:
base_url = self.get_endpoint(**endpoint_filter)
if not base_url:
raise exceptions.EndpointNotFound()
url = '%s/%s' % (base_url.rstrip('/'), url.lstrip('/'))
if self.cert:
kwargs.setdefault('cert', self.cert)
if self.timeout is not None:
kwargs.setdefault('timeout', self.timeout)
if user_agent:
headers['User-Agent'] = user_agent
elif self.user_agent:
user_agent = headers.setdefault('User-Agent', self.user_agent)
else:
user_agent = headers.setdefault('User-Agent', USER_AGENT)
if self.original_ip:
headers.setdefault('Forwarded',
'for=%s;by=%s' % (self.original_ip, user_agent))
if json is not None:
headers['Content-Type'] = 'application/json'
kwargs['data'] = jsonutils.dumps(json)
kwargs.setdefault('verify', self.verify)
string_parts = ['curl -i']
# NOTE(jamielennox): None means let requests do its default validation
# so we need to actually check that this is False.
if self.verify is False:
string_parts.append('--insecure')
if method:
string_parts.extend(['-X', method])
string_parts.append(url)
if headers:
for header in six.iteritems(headers):
string_parts.append('-H "%s: %s"' % header)
try:
string_parts.append("-d '%s'" % kwargs['data'])
except KeyError:
pass
_logger.debug('REQ: %s', ' '.join(string_parts))
# Force disable requests redirect handling. We will manage this below.
kwargs['allow_redirects'] = False
if redirect is None:
redirect = self.redirect
resp = self._send_request(url, method, redirect, **kwargs)
# NOTE(jamielennox): we create a tuple here to be the same as what is
# returned by the requests library.
resp.history = tuple(resp.history)
if resp.status_code >= 400:
_logger.debug('Request returned failure status: %s',
resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp
**request_kwargs)
self.http_log((url, method,), request_kwargs, resp, body)
if body:
try:
body = json.loads(body)
except ValueError, e:
_logger.debug("Could not decode JSON from body: %s" % body)
else:
_logger.debug("No body was returned.")
body = None
if resp.status in (400, 401, 403, 404, 408, 409, 413, 500, 501):
_logger.exception("Request returned failure status.")
raise exceptions.from_response(resp, body)
elif resp.status in (301, 302, 305):
# Redirected. Reissue the request to the new location.
return self.request(resp['location'], method, **kwargs)
return resp, body
def _cs_request(self, url, method, **kwargs):
if not self.management_url:
self.authenticate()
kwargs.setdefault('headers', {})
if self.auth_token:
kwargs['headers']['X-Auth-Token'] = self.auth_token
# Perform the request once. If we get a 401 back then it
url,
method,
), request_kwargs, resp, body)
if body:
try:
body = json.loads(body)
except ValueError, e:
_logger.debug("Could not decode JSON from body: %s" % body)
else:
_logger.debug("No body was returned.")
body = None
if resp.status in (400, 401, 403, 404, 408, 409, 413, 500, 501):
_logger.exception("Request returned failure status.")
raise exceptions.from_response(resp, body)
elif resp.status in (301, 302, 305):
# Redirected. Reissue the request to the new location.
return self.request(resp['location'], method, **kwargs)
return resp, body
def _cs_request(self, url, method, **kwargs):
if not self.management_url:
self.authenticate()
kwargs.setdefault('headers', {})
if self.auth_token and self.auth_token != self.password:
kwargs['headers']['X-Auth-Token'] = self.auth_token
# Perform the request once. If we get a 401 back then it
def request(self,
url,
method,
json=None,
original_ip=None,
user_agent=None,
redirect=None,
authenticated=None,
**kwargs):
"""Send an HTTP request with the specified characteristics.
Wrapper around `requests.Session.request` to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
Arguments that are not handled are passed through to the requests
library.
:param string url: Fully qualified URL of HTTP request
:param string method: The http method to use. (eg. 'GET', 'POST')
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param dict headers: Headers to be included in the request. (optional)
:param json: Some data to be represented as JSON. (optional)
:param string user_agent: A user_agent to use for the request. If
present will override one present in headers.
(optional)
:param int/bool redirect: the maximum number of redirections that
can be followed by a request. Either an
integer for a specific count or True/False
for forever/never. (optional)
:param bool authenticated: True if a token should be attached to this
request, False if not or None for attach if
an auth_plugin is available.
(optional, defaults to None)
:param kwargs: any other parameter that can be passed to
requests.Session.request (such as `headers`). Except:
'data' will be overwritten by the data in 'json' param.
'allow_redirects' is ignored as redirects are handled
by the session.
:raises exceptions.ClientException: For connection failure, or to
indicate an error response code.
:returns: The response to the request.
"""
headers = kwargs.setdefault('headers', dict())
if authenticated is None:
authenticated = self.auth is not None
if authenticated:
token = self.get_token()
if not token:
raise exceptions.AuthorizationFailure("No token Available")
headers['X-Auth-Token'] = token
if self.cert:
kwargs.setdefault('cert', self.cert)
if self.timeout is not None:
kwargs.setdefault('timeout', self.timeout)
if user_agent:
headers['User-Agent'] = user_agent
elif self.user_agent:
user_agent = headers.setdefault('User-Agent', self.user_agent)
else:
user_agent = headers.setdefault('User-Agent', USER_AGENT)
if self.original_ip:
headers.setdefault('Forwarded',
'for=%s;by=%s' % (self.original_ip, user_agent))
if json is not None:
headers['Content-Type'] = 'application/json'
kwargs['data'] = jsonutils.dumps(json)
kwargs.setdefault('verify', self.verify)
string_parts = ['curl -i']
# NOTE(jamielennox): None means let requests do its default validation
# so we need to actually check that this is False.
if self.verify is False:
string_parts.append('--insecure')
if method:
string_parts.extend(['-X', method])
string_parts.append(url)
if headers:
for header in six.iteritems(headers):
string_parts.append('-H "%s: %s"' % header)
try:
string_parts.append("-d '%s'" % kwargs['data'])
except KeyError:
pass
_logger.debug('REQ: %s', ' '.join(string_parts))
# Force disable requests redirect handling. We will manage this below.
kwargs['allow_redirects'] = False
if redirect is None:
redirect = self.redirect
resp = self._send_request(url, method, redirect, **kwargs)
# NOTE(jamielennox): we create a tuple here to be the same as what is
# returned by the requests library.
resp.history = tuple(resp.history)
if resp.status_code >= 400:
_logger.debug('Request returned failure status: %s',
resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp
def request(self, url, method, json=None, original_ip=None, debug=None,
logger=None, user_agent=None, **kwargs):
"""Send an HTTP request with the specified characteristics.
Wrapper around `requests.Session.request` to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
Arguments that are not handled are passed through to the requests
library.
:param string url: Fully qualified URL of HTTP request
:param string method: The http method to use. (eg. 'GET', 'POST')
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param dict headers: Headers to be included in the request. (optional)
:param bool debug: Enable debug logging. (Defaults to False)
:param kwargs: any other parameter that can be passed to
requests.Session.request (such as `headers`) or `json`
that will be encoded as JSON and used as `data` argument
:param logging.Logger logger: A logger to output to. (optional)
:param json: Some data to be represented as JSON. (optional)
:param string user_agent: A user_agent to use for the request. If
present will override one present in headers.
(optional)
:raises exceptions.ClientException: For connection failure, or to
indicate an error response code.
:returns: The response to the request.
"""
headers = kwargs.setdefault('headers', dict())
if self.cert:
kwargs.setdefault('cert', self.cert)
if self.timeout is not None:
kwargs.setdefault('timeout', self.timeout)
if user_agent:
headers['User-Agent'] = user_agent
elif self.user_agent:
user_agent = headers.setdefault('User-Agent', self.user_agent)
else:
user_agent = headers.setdefault('User-Agent', USER_AGENT)
if self.original_ip:
headers.setdefault('Forwarded',
'for=%s;by=%s' % (self.original_ip, user_agent))
if json is not None:
headers['Content-Type'] = 'application/json'
kwargs['data'] = jsonutils.dumps(json)
if not logger:
logger = _logger
if debug is None:
debug = self.debug
kwargs.setdefault('verify', self.verify)
if debug:
string_parts = ['curl -i']
if method:
string_parts.extend([' -X ', method])
string_parts.extend([' ', url])
if headers:
for header in six.iteritems(headers):
string_parts.append(' -H "%s: %s"' % header)
logger.debug('REQ: %s', ''.join(string_parts))
data = kwargs.get('data')
if data:
logger.debug('REQ BODY: %s', data)
try:
resp = self.session.request(method, url, **kwargs)
except requests.exceptions.SSLError:
msg = 'SSL exception connecting to %s' % url
raise exceptions.SSLError(msg)
except requests.exceptions.Timeout:
msg = 'Request to %s timed out' % url
raise exceptions.Timeout(msg)
except requests.exceptions.ConnectionError:
msg = 'Unable to establish connection to %s' % url
raise exceptions.ConnectionError(msg)
if debug:
logger.debug('RESP: [%s] %s\nRESP BODY: %s\n',
resp.status_code, resp.headers, resp.text)
if resp.status_code >= 400:
logger.debug('Request returned failure status: %s',
resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp
def request(url, method='GET', headers=None, original_ip=None, debug=False,
logger=None, **kwargs):
"""Perform a http request with standard settings.
A wrapper around requests.request that adds standard headers like
User-Agent and provides optional debug logging of the request.
Arguments that are not handled are passed through to the requests library.
:param string url: The url to make the request of.
:param string method: The http method to use. (eg. 'GET', 'POST')
:param dict headers: Headers to be included in the request. (optional)
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param bool debug: Enable debug logging. (Defaults to False)
:param logging.Logger logger: A logger to output to. (optional)
:raises exceptions.ClientException: For connection failure, or to indicate
an error response code.
:returns: The response to the request.
"""
if not headers:
headers = dict()
if not logger:
logger = _logger
headers.setdefault('User-Agent', USER_AGENT)
if original_ip:
headers['Forwarded'] = "for=%s;by=%s" % (original_ip, USER_AGENT)
if debug:
string_parts = ['curl -i']
if method:
string_parts.append(' -X %s' % method)
string_parts.append(' %s' % url)
if headers:
for header in headers.iteritems():
string_parts.append(' -H "%s: %s"' % header)
logger.debug("REQ: %s" % "".join(string_parts))
data = kwargs.get('data')
if data:
logger.debug("REQ BODY: %s\n" % data)
try:
resp = requests.request(
method,
url,
headers=headers,
**kwargs)
except requests.ConnectionError:
msg = 'Unable to establish connection to %s' % url
raise exceptions.ClientException(msg)
if debug:
logger.debug("RESP: [%s] %s\nRESP BODY: %s\n",
resp.status_code, resp.headers, resp.text)
if resp.status_code >= 400:
logger.debug("Request returned failure status: %s",
resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp
def request(
self,
url,
method,
json=None,
original_ip=None,
user_agent=None,
redirect=None,
authenticated=None,
endpoint_filter=None,
auth=None,
requests_auth=None,
raise_exc=True,
allow_reauth=True,
log=True,
endpoint_override=None,
connect_retries=0,
logger=_logger,
**kwargs
):
"""Send an HTTP request with the specified characteristics.
Wrapper around `requests.Session.request` to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
Arguments that are not handled are passed through to the requests
library.
:param string url: Path or fully qualified URL of HTTP request. If only
a path is provided then endpoint_filter must also be
provided such that the base URL can be determined.
If a fully qualified URL is provided then
endpoint_filter will be ignored.
:param string method: The http method to use. (e.g. 'GET', 'POST')
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param dict headers: Headers to be included in the request. (optional)
:param json: Some data to be represented as JSON. (optional)
:param string user_agent: A user_agent to use for the request. If
present will override one present in headers.
(optional)
:param int/bool redirect: the maximum number of redirections that
can be followed by a request. Either an
integer for a specific count or True/False
for forever/never. (optional)
:param int connect_retries: the maximum number of retries that should
be attempted for connection errors.
(optional, defaults to 0 - never retry).
:param bool authenticated: True if a token should be attached to this
request, False if not or None for attach if
an auth_plugin is available.
(optional, defaults to None)
:param dict endpoint_filter: Data to be provided to an auth plugin with
which it should be able to determine an
endpoint to use for this request. If not
provided then URL is expected to be a
fully qualified URL. (optional)
:param str endpoint_override: The URL to use instead of looking up the
endpoint in the auth plugin. This will be
ignored if a fully qualified URL is
provided but take priority over an
endpoint_filter. (optional)
:param auth: The auth plugin to use when authenticating this request.
This will override the plugin that is attached to the
session (if any). (optional)
:type auth: :py:class:`keystoneclient.auth.base.BaseAuthPlugin`
:param requests_auth: A requests library auth plugin that cannot be
passed via kwarg because the `auth` kwarg
collides with our own auth plugins. (optional)
:type requests_auth: :py:class:`requests.auth.AuthBase`
:param bool raise_exc: If True then raise an appropriate exception for
failed HTTP requests. If False then return the
request object. (optional, default True)
:param bool allow_reauth: Allow fetching a new token and retrying the
request on receiving a 401 Unauthorized
response. (optional, default True)
:param bool log: If True then log the request and response data to the
debug log. (optional, default True)
:param logger: The logger object to use to log request and responses.
If not provided the keystoneclient.session default
logger will be used.
:type logger: logging.Logger
:param kwargs: any other parameter that can be passed to
requests.Session.request (such as `headers`). Except:
'data' will be overwritten by the data in 'json' param.
'allow_redirects' is ignored as redirects are handled
by the session.
:raises keystoneclient.exceptions.ClientException: For connection
failure, or to indicate an error response code.
:returns: The response to the request.
"""
headers = kwargs.setdefault("headers", dict())
if authenticated is None:
authenticated = bool(auth or self.auth)
if authenticated:
auth_headers = self.get_auth_headers(auth)
if auth_headers is None:
msg = _("No valid authentication is available")
raise exceptions.AuthorizationFailure(msg)
headers.update(auth_headers)
if osprofiler_web:
headers.update(osprofiler_web.get_trace_id_headers())
# if we are passed a fully qualified URL and an endpoint_filter we
# should ignore the filter. This will make it easier for clients who
# want to overrule the default endpoint_filter data added to all client
# requests. We check fully qualified here by the presence of a host.
if not urllib.parse.urlparse(url).netloc:
base_url = None
if endpoint_override:
base_url = endpoint_override
elif endpoint_filter:
base_url = self.get_endpoint(auth, **endpoint_filter)
if not base_url:
service_type = (endpoint_filter or {}).get("service_type", "unknown")
msg = _("Endpoint for %s service") % service_type
raise exceptions.EndpointNotFound(msg)
url = "%s/%s" % (base_url.rstrip("/"), url.lstrip("/"))
if self.cert:
kwargs.setdefault("cert", self.cert)
if self.timeout is not None:
kwargs.setdefault("timeout", self.timeout)
if user_agent:
headers["User-Agent"] = user_agent
elif self.user_agent:
user_agent = headers.setdefault("User-Agent", self.user_agent)
else:
user_agent = headers.setdefault("User-Agent", USER_AGENT)
if self.original_ip:
headers.setdefault("Forwarded", "for=%s;by=%s" % (self.original_ip, user_agent))
if json is not None:
headers["Content-Type"] = "application/json"
kwargs["data"] = jsonutils.dumps(json)
kwargs.setdefault("verify", self.verify)
if requests_auth:
kwargs["auth"] = requests_auth
if log:
self._http_log_request(url, method=method, data=kwargs.get("data"), headers=headers, logger=logger)
# Force disable requests redirect handling. We will manage this below.
kwargs["allow_redirects"] = False
if redirect is None:
redirect = self.redirect
send = functools.partial(self._send_request, url, method, redirect, log, logger, connect_retries)
try:
connection_params = self.get_auth_connection_params(auth=auth)
except exceptions.MissingAuthPlugin:
# NOTE(jamielennox): If we've gotten this far without an auth
# plugin then we should be happy with allowing no additional
# connection params. This will be the typical case for plugins
# anyway.
pass
else:
if connection_params:
kwargs.update(connection_params)
resp = send(**kwargs)
# handle getting a 401 Unauthorized response by invalidating the plugin
# and then retrying the request. This is only tried once.
if resp.status_code == 401 and authenticated and allow_reauth:
if self.invalidate(auth):
auth_headers = self.get_auth_headers(auth)
if auth_headers is not None:
headers.update(auth_headers)
resp = send(**kwargs)
if raise_exc and resp.status_code >= 400:
logger.debug("Request returned failure status: %s", resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp
def _retrieve_data_from_keystone(redis_client, url, tenant, token,
blacklist_ttl, max_cache_life):
"""Retrieve the authentication data from OpenStack Keystone
:param redis_client: redis.Redis object connected to the redis cache
:param url: Keystone Identity URL to authenticate against
:param tenant: tenant id of user data to retrieve
:param token: auth_token for the tenant_id
:param blacklist_ttl: time in milliseconds for blacklisting failed tokens
:param max_cache_life: time in seconds for the maximum time a cache entry
should remain in the cache of valid data
:returns: a keystoneclient.access.AccessInfo on success or None on error
"""
try:
# Try to authenticate the user and get the user information using
# only the data provided, no special administrative tokens required.
# When using the alternative validation method, the service catalog
# identity does not return a service catalog for valid tokens.
if get_conf().alternate_validation is True:
_url = url.rstrip('/') + '/tokens'
validation_url = _url + '/{0}'.format(token)
headers = {
'Accept': 'application/json',
'X-Auth-Token': token
}
resp = requests.get(validation_url, headers=headers)
if resp.status_code >= 400:
LOG.debug('Request returned failure status: {0}'.format(
resp.status_code))
raise exceptions.from_response(resp, 'GET', _url)
try:
resp_data = resp.json()['access']
except (KeyError, ValueError):
raise exceptions.InvalidResponse(response=resp)
access_info = access.AccessInfoV2(**resp_data)
else:
keystone = keystonev2_client.Client(tenant_id=tenant,
token=token,
auth_url=url)
access_info = keystone.get_raw_token_from_identity_service(
auth_url=url, tenant_id=tenant, token=token)
# cache the data so it is easier to access next time
_send_data_to_cache(redis_client, url, access_info, max_cache_life)
return access_info
except (exceptions.AuthorizationFailure, exceptions.Unauthorized) as ex:
# re-raise 413 here and later on respond with 503
if 'HTTP 413' in str(ex):
raise exceptions.RequestEntityTooLarge(
method='POST',
url=url,
http_status=413
)
# Provided data was invalid and authorization failed
msg = 'Failed to authenticate against {0} - {1}'.format(
url,
str(ex)
)
LOG.debug(msg)
# Blacklist the token
_blacklist_token(redis_client, token, blacklist_ttl)
return None
except exceptions.RequestEntityTooLarge:
LOG.debug('Request entity too large error from authentication server.')
raise
except Exception as ex:
# Provided data was invalid or something else went wrong
msg = 'Failed to authenticate against {0} - {1}'.format(
url,
str(ex)
)
LOG.debug(msg)
return None
def request(url,
method='GET',
headers=None,
original_ip=None,
debug=False,
logger=None,
**kwargs):
"""Perform a http request with standard settings.
A wrapper around requests.request that adds standard headers like
User-Agent and provides optional debug logging of the request.
Arguments that are not handled are passed through to the requests library.
:param string url: The url to make the request of.
:param string method: The http method to use. (eg. 'GET', 'POST')
:param dict headers: Headers to be included in the request. (optional)
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param bool debug: Enable debug logging. (Defaults to False)
:param logging.Logger logger: A logger to output to. (optional)
:raises exceptions.ClientException: For connection failure, or to indicate
an error response code.
:returns: The response to the request.
"""
if not headers:
headers = dict()
if not logger:
logger = _logger
headers.setdefault('User-Agent', USER_AGENT)
if original_ip:
headers['Forwarded'] = "for=%s;by=%s" % (original_ip, USER_AGENT)
if debug:
string_parts = ['curl -i']
if method:
string_parts.append(' -X %s' % method)
string_parts.append(' %s' % url)
if headers:
for header in headers.iteritems():
string_parts.append(' -H "%s: %s"' % header)
logger.debug("REQ: %s" % "".join(string_parts))
data = kwargs.get('data')
if data:
logger.debug("REQ BODY: %s\n" % data)
try:
resp = requests.request(method, url, headers=headers, **kwargs)
except requests.ConnectionError:
msg = 'Unable to establish connection to %s' % url
raise exceptions.ClientException(msg)
if debug:
logger.debug("RESP: [%s] %s\nRESP BODY: %s\n", resp.status_code,
resp.headers, resp.text)
if resp.status_code >= 400:
logger.debug("Request returned failure status: %s", resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp
def request(self, url, method, json=None, original_ip=None,
user_agent=None, redirect=None, authenticated=None,
endpoint_filter=None, auth=None, requests_auth=None,
raise_exc=True, allow_reauth=True, log=True, **kwargs):
"""Send an HTTP request with the specified characteristics.
Wrapper around `requests.Session.request` to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
Arguments that are not handled are passed through to the requests
library.
:param string url: Path or fully qualified URL of HTTP request. If only
a path is provided then endpoint_filter must also be
provided such that the base URL can be determined.
If a fully qualified URL is provided then
endpoint_filter will be ignored.
:param string method: The http method to use. (e.g. 'GET', 'POST')
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param dict headers: Headers to be included in the request. (optional)
:param json: Some data to be represented as JSON. (optional)
:param string user_agent: A user_agent to use for the request. If
present will override one present in headers.
(optional)
:param int/bool redirect: the maximum number of redirections that
can be followed by a request. Either an
integer for a specific count or True/False
for forever/never. (optional)
:param bool authenticated: True if a token should be attached to this
request, False if not or None for attach if
an auth_plugin is available.
(optional, defaults to None)
:param dict endpoint_filter: Data to be provided to an auth plugin with
which it should be able to determine an
endpoint to use for this request. If not
provided then URL is expected to be a
fully qualified URL. (optional)
:param auth: The auth plugin to use when authenticating this request.
This will override the plugin that is attached to the
session (if any). (optional)
:type auth: :class:`keystoneclient.auth.base.BaseAuthPlugin`
:param requests_auth: A requests library auth plugin that cannot be
passed via kwarg because the `auth` kwarg
collides with our own auth plugins. (optional)
:type requests_auth: :class:`requests.auth.AuthBase`
:param bool raise_exc: If True then raise an appropriate exception for
failed HTTP requests. If False then return the
request object. (optional, default True)
:param bool allow_reauth: Allow fetching a new token and retrying the
request on receiving a 401 Unauthorized
response. (optional, default True)
:param bool log: If True then log the request and response data to the
debug log. (optional, default True)
:param kwargs: any other parameter that can be passed to
requests.Session.request (such as `headers`). Except:
'data' will be overwritten by the data in 'json' param.
'allow_redirects' is ignored as redirects are handled
by the session.
:raises exceptions.ClientException: For connection failure, or to
indicate an error response code.
:returns: The response to the request.
"""
headers = kwargs.setdefault('headers', dict())
if authenticated is None:
authenticated = bool(auth or self.auth)
if authenticated:
token = self.get_token(auth)
if not token:
raise exceptions.AuthorizationFailure("No token Available")
headers['X-Auth-Token'] = token
if osprofiler_web:
headers.update(osprofiler_web.get_trace_id_headers())
# if we are passed a fully qualified URL and an endpoint_filter we
# should ignore the filter. This will make it easier for clients who
# want to overrule the default endpoint_filter data added to all client
# requests. We check fully qualified here by the presence of a host.
url_data = urllib.parse.urlparse(url)
if endpoint_filter and not url_data.netloc:
base_url = self.get_endpoint(auth, **endpoint_filter)
if not base_url:
raise exceptions.EndpointNotFound()
url = '%s/%s' % (base_url.rstrip('/'), url.lstrip('/'))
if self.cert:
kwargs.setdefault('cert', self.cert)
if self.timeout is not None:
kwargs.setdefault('timeout', self.timeout)
if user_agent:
headers['User-Agent'] = user_agent
elif self.user_agent:
user_agent = headers.setdefault('User-Agent', self.user_agent)
else:
user_agent = headers.setdefault('User-Agent', USER_AGENT)
if self.original_ip:
headers.setdefault('Forwarded',
'for=%s;by=%s' % (self.original_ip, user_agent))
if json is not None:
headers['Content-Type'] = 'application/json'
kwargs['data'] = jsonutils.dumps(json)
kwargs.setdefault('verify', self.verify)
if requests_auth:
kwargs['auth'] = requests_auth
if log:
self._http_log_request(url, method=method,
data=kwargs.get('data'),
headers=headers)
# Force disable requests redirect handling. We will manage this below.
kwargs['allow_redirects'] = False
if redirect is None:
redirect = self.redirect
resp = self._send_request(url, method, redirect, log, **kwargs)
# handle getting a 401 Unauthorized response by invalidating the plugin
# and then retrying the request. This is only tried once.
if resp.status_code == 401 and authenticated and allow_reauth:
if self.invalidate(auth):
token = self.get_token(auth)
if token:
headers['X-Auth-Token'] = token
resp = self._send_request(url, method, redirect, log,
**kwargs)
if raise_exc and resp.status_code >= 400:
_logger.debug('Request returned failure status: %s',
resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp
def request(self,
url,
method,
json=None,
original_ip=None,
user_agent=None,
redirect=None,
authenticated=None,
endpoint_filter=None,
auth=None,
requests_auth=None,
raise_exc=True,
**kwargs):
"""Send an HTTP request with the specified characteristics.
Wrapper around `requests.Session.request` to handle tasks such as
setting headers, JSON encoding/decoding, and error handling.
Arguments that are not handled are passed through to the requests
library.
:param string url: Path or fully qualified URL of HTTP request. If only
a path is provided then endpoint_filter must also be
provided such that the base URL can be determined.
If a fully qualified URL is provided then
endpoint_filter will be ignored.
:param string method: The http method to use. (e.g. 'GET', 'POST')
:param string original_ip: Mark this request as forwarded for this ip.
(optional)
:param dict headers: Headers to be included in the request. (optional)
:param json: Some data to be represented as JSON. (optional)
:param string user_agent: A user_agent to use for the request. If
present will override one present in headers.
(optional)
:param int/bool redirect: the maximum number of redirections that
can be followed by a request. Either an
integer for a specific count or True/False
for forever/never. (optional)
:param bool authenticated: True if a token should be attached to this
request, False if not or None for attach if
an auth_plugin is available.
(optional, defaults to None)
:param dict endpoint_filter: Data to be provided to an auth plugin with
which it should be able to determine an
endpoint to use for this request. If not
provided then URL is expected to be a
fully qualified URL. (optional)
:param auth: The auth plugin to use when authenticating this request.
This will override the plugin that is attached to the
session (if any). (optional)
:type auth: :class:`keystoneclient.auth.base.BaseAuthPlugin`
:param requests_auth: A requests library auth plugin that cannot be
passed via kwarg because the `auth` kwarg
collides with our own auth plugins. (optional)
:type requests_auth: :class:`requests.auth.AuthBase`
:param bool raise_exc: If True then raise an appropriate exception for
failed HTTP requests. If False then return the
request object. (optional, default True)
:param kwargs: any other parameter that can be passed to
requests.Session.request (such as `headers`). Except:
'data' will be overwritten by the data in 'json' param.
'allow_redirects' is ignored as redirects are handled
by the session.
:raises exceptions.ClientException: For connection failure, or to
indicate an error response code.
:returns: The response to the request.
"""
headers = kwargs.setdefault('headers', dict())
if authenticated is None:
authenticated = bool(auth or self.auth)
if authenticated:
token = self.get_token(auth)
if not token:
raise exceptions.AuthorizationFailure("No token Available")
headers['X-Auth-Token'] = token
# if we are passed a fully qualified URL and an endpoint_filter we
# should ignore the filter. This will make it easier for clients who
# want to overrule the default endpoint_filter data added to all client
# requests. We check fully qualified here by the presence of a host.
url_data = urllib.parse.urlparse(url)
if endpoint_filter and not url_data.netloc:
base_url = self.get_endpoint(auth, **endpoint_filter)
if not base_url:
raise exceptions.EndpointNotFound()
url = '%s/%s' % (base_url.rstrip('/'), url.lstrip('/'))
if self.cert:
kwargs.setdefault('cert', self.cert)
if self.timeout is not None:
kwargs.setdefault('timeout', self.timeout)
if user_agent:
headers['User-Agent'] = user_agent
elif self.user_agent:
user_agent = headers.setdefault('User-Agent', self.user_agent)
else:
user_agent = headers.setdefault('User-Agent', USER_AGENT)
if self.original_ip:
headers.setdefault('Forwarded',
'for=%s;by=%s' % (self.original_ip, user_agent))
if json is not None:
headers['Content-Type'] = 'application/json'
kwargs['data'] = jsonutils.dumps(json)
kwargs.setdefault('verify', self.verify)
if requests_auth:
kwargs['auth'] = requests_auth
string_parts = ['curl -i']
# NOTE(jamielennox): None means let requests do its default validation
# so we need to actually check that this is False.
if self.verify is False:
string_parts.append('--insecure')
if method:
string_parts.extend(['-X', method])
string_parts.append(url)
if headers:
for header in six.iteritems(headers):
string_parts.append('-H "%s: %s"' % header)
try:
string_parts.append("-d '%s'" % kwargs['data'])
except KeyError:
pass
_logger.debug('REQ: %s', ' '.join(string_parts))
# Force disable requests redirect handling. We will manage this below.
kwargs['allow_redirects'] = False
if redirect is None:
redirect = self.redirect
resp = self._send_request(url, method, redirect, **kwargs)
if raise_exc and resp.status_code >= 400:
_logger.debug('Request returned failure status: %s',
resp.status_code)
raise exceptions.from_response(resp, method, url)
return resp