def process_request(self, request): if 'profile_page' in request.COOKIES: hmac_key = PROFILER_SETTINGS.get('keys')[0] profiler.init(hmac_key) for hdr_key, hdr_value in web.get_trace_id_headers().items(): request.META[hdr_key] = hdr_value return None
def test_get_trace_id_headers(self): profiler.init("key", base_id="y", parent_id="z") headers = web.get_trace_id_headers() self.assertEqual(sorted(headers.keys()), sorted(["X-Trace-Info", "X-Trace-HMAC"])) trace_info = utils.signed_unpack(headers["X-Trace-Info"], headers["X-Trace-HMAC"], ["key"]) self.assertIn("hmac_key", trace_info) self.assertEqual("key", trace_info.pop("hmac_key")) self.assertEqual({"parent_id": "z", "base_id": "y"}, trace_info)
def update_trace_headers(keys, **kwargs): trace_headers = web.get_trace_id_headers() trace_info = utils.signed_unpack( trace_headers[web.X_TRACE_INFO], trace_headers[web.X_TRACE_HMAC], keys) trace_info.update(kwargs) p = profiler.get() trace_data = utils.signed_pack(trace_info, p.hmac_key) return json.dumps({web.X_TRACE_INFO: trace_data[0], web.X_TRACE_HMAC: trace_data[1]})
def process_request(self, request): if self.is_async_profiling(request): for src_header, dst_header in self.profiler_headers: request.META[dst_header] = request.META.get(src_header) return None if 'profile_page' in request.COOKIES: hmac_key = PROFILER_CONF.get('keys')[0] profiler.init(hmac_key) for hdr_key, hdr_value in web.get_trace_id_headers().items(): request.META[hdr_key] = hdr_value return None
def update_trace_headers(keys, **kwargs): trace_headers = web.get_trace_id_headers() trace_info = utils.signed_unpack( trace_headers[web.X_TRACE_INFO], trace_headers[web.X_TRACE_HMAC], keys) trace_info.update(kwargs) p = profiler.get() trace_data = utils.signed_pack(trace_info, p.hmac_key) trace_data = [key.decode() if isinstance(key, bytes) else key for key in trace_data] return json.dumps({web.X_TRACE_INFO: trace_data[0], web.X_TRACE_HMAC: trace_data[1]})
def process_request(self, request): if self.is_async_profiling(request): for src_header, dst_header in self.profiler_headers: request.META[dst_header] = request.META.get(src_header) return None if 'profile_page' in request.COOKIES: hmac_key = horizon_settings.get_dict_config( 'OPENSTACK_PROFILER', 'keys')[0] profiler.init(hmac_key) for hdr_key, hdr_value in web.get_trace_id_headers().items(): request.META[hdr_key] = hdr_value return None
def update_trace_headers(keys, **kwargs): trace_headers = web.get_trace_id_headers() trace_info = utils.signed_unpack( trace_headers[web.X_TRACE_INFO], trace_headers[web.X_TRACE_HMAC], keys) trace_info.update(kwargs) p = profiler.get() trace_data = utils.signed_pack(trace_info, p.hmac_key) if six.PY3: trace_data = [key.decode() if isinstance(key, six.binary_type) else key for key in trace_data] return json.dumps({web.X_TRACE_INFO: trace_data[0], web.X_TRACE_HMAC: trace_data[1]})
def request(self, url, method, **kwargs): kwargs.setdefault('headers', kwargs.get('headers', {})) # NOTE(sileht): The standard call raises errors from # keystoneauth, where we need to raise the aodhclient errors. raise_exc = kwargs.pop('raise_exc', True) kwargs['headers'].update(web.get_trace_id_headers()) resp = super(SessionClient, self).request(url, method, raise_exc=False, **kwargs) if raise_exc and resp.status_code >= 400: raise exceptions.from_response(resp, url, method) return resp
def test_get_trace_id_headers_no_profiler(self, mock_get_profiler): mock_get_profiler.return_value = False headers = web.get_trace_id_headers() self.assertEqual(headers, {})
def request(self, url, method, json=None, original_ip=None, user_agent=None, redirect=None, authenticated=None, endpoint_filter=None, auth=None, requests_auth=None, raise_exc=True, allow_reauth=True, log=True, endpoint_override=None, connect_retries=0, logger=_logger, allow={}, **kwargs): """Send an HTTP request with the specified characteristics. Wrapper around `requests.Session.request` to handle tasks such as setting headers, JSON encoding/decoding, and error handling. Arguments that are not handled are passed through to the requests library. :param str url: Path or fully qualified URL of HTTP request. If only a path is provided then endpoint_filter must also be provided such that the base URL can be determined. If a fully qualified URL is provided then endpoint_filter will be ignored. :param str method: The http method to use. (e.g. 'GET', 'POST') :param str original_ip: Mark this request as forwarded for this ip. (optional) :param dict headers: Headers to be included in the request. (optional) :param json: Some data to be represented as JSON. (optional) :param str user_agent: A user_agent to use for the request. If present will override one present in headers. (optional) :param int/bool redirect: the maximum number of redirections that can be followed by a request. Either an integer for a specific count or True/False for forever/never. (optional) :param int connect_retries: the maximum number of retries that should be attempted for connection errors. (optional, defaults to 0 - never retry). :param bool authenticated: True if a token should be attached to this request, False if not or None for attach if an auth_plugin is available. (optional, defaults to None) :param dict endpoint_filter: Data to be provided to an auth plugin with which it should be able to determine an endpoint to use for this request. If not provided then URL is expected to be a fully qualified URL. (optional) :param str endpoint_override: The URL to use instead of looking up the endpoint in the auth plugin. This will be ignored if a fully qualified URL is provided but take priority over an endpoint_filter. This string may contain the values ``%(project_id)s`` and ``%(user_id)s`` to have those values replaced by the project_id/user_id of the current authentication. (optional) :param auth: The auth plugin to use when authenticating this request. This will override the plugin that is attached to the session (if any). (optional) :type auth: keystoneauth1.plugin.BaseAuthPlugin :param requests_auth: A requests library auth plugin that cannot be passed via kwarg because the `auth` kwarg collides with our own auth plugins. (optional) :type requests_auth: :py:class:`requests.auth.AuthBase` :param bool raise_exc: If True then raise an appropriate exception for failed HTTP requests. If False then return the request object. (optional, default True) :param bool allow_reauth: Allow fetching a new token and retrying the request on receiving a 401 Unauthorized response. (optional, default True) :param bool log: If True then log the request and response data to the debug log. (optional, default True) :param logger: The logger object to use to log request and responses. If not provided the keystoneauth1.session default logger will be used. :type logger: logging.Logger :param dict allow: Extra filters to pass when discovering API versions. (optional) :param kwargs: any other parameter that can be passed to :meth:`requests.Session.request` (such as `headers`). Except: - `data` will be overwritten by the data in the `json` param. - `allow_redirects` is ignored as redirects are handled by the session. :raises keystoneauth1.exceptions.base.ClientException: For connection failure, or to indicate an error response code. :returns: The response to the request. """ headers = kwargs.setdefault('headers', dict()) if authenticated is None: authenticated = bool(auth or self.auth) if authenticated: auth_headers = self.get_auth_headers(auth) if auth_headers is None: msg = 'No valid authentication is available' raise exceptions.AuthorizationFailure(msg) headers.update(auth_headers) if osprofiler_web: headers.update(osprofiler_web.get_trace_id_headers()) # if we are passed a fully qualified URL and an endpoint_filter we # should ignore the filter. This will make it easier for clients who # want to overrule the default endpoint_filter data added to all client # requests. We check fully qualified here by the presence of a host. if not urllib.parse.urlparse(url).netloc: base_url = None if endpoint_override: base_url = endpoint_override % _StringFormatter(self, auth) elif endpoint_filter: base_url = self.get_endpoint(auth, allow=allow, **endpoint_filter) if not base_url: raise exceptions.EndpointNotFound() url = '%s/%s' % (base_url.rstrip('/'), url.lstrip('/')) if self.cert: kwargs.setdefault('cert', self.cert) if self.timeout is not None: kwargs.setdefault('timeout', self.timeout) if user_agent: headers['User-Agent'] = user_agent elif self.user_agent: user_agent = headers.setdefault('User-Agent', self.user_agent) else: user_agent = headers.setdefault('User-Agent', DEFAULT_USER_AGENT) if self.original_ip: headers.setdefault('Forwarded', 'for=%s;by=%s' % (self.original_ip, user_agent)) if json is not None: headers['Content-Type'] = 'application/json' kwargs['data'] = self._json.encode(json) kwargs.setdefault('verify', self.verify) if requests_auth: kwargs['auth'] = requests_auth # Query parameters that are included in the url string will # be logged properly, but those sent in the `params` parameter # (which the requests library handles) need to be explicitly # picked out so they can be included in the URL that gets loggged. query_params = kwargs.get('params', dict()) if log: self._http_log_request(url, method=method, data=kwargs.get('data'), headers=headers, query_params=query_params, logger=logger) # Force disable requests redirect handling. We will manage this below. kwargs['allow_redirects'] = False if redirect is None: redirect = self.redirect send = functools.partial(self._send_request, url, method, redirect, log, logger, connect_retries) try: connection_params = self.get_auth_connection_params(auth=auth) except exceptions.MissingAuthPlugin: # NOTE(jamielennox): If we've gotten this far without an auth # plugin then we should be happy with allowing no additional # connection params. This will be the typical case for plugins # anyway. pass else: if connection_params: kwargs.update(connection_params) resp = send(**kwargs) # handle getting a 401 Unauthorized response by invalidating the plugin # and then retrying the request. This is only tried once. if resp.status_code == 401 and authenticated and allow_reauth: if self.invalidate(auth): auth_headers = self.get_auth_headers(auth) if auth_headers is not None: headers.update(auth_headers) resp = send(**kwargs) if raise_exc and resp.status_code >= 400: logger.debug('Request returned failure status: %s', resp.status_code) raise exceptions.from_response(resp, method, url) return resp
def request(self, url, method, json=None, original_ip=None, user_agent=None, redirect=None, authenticated=None, endpoint_filter=None, auth=None, requests_auth=None, raise_exc=True, allow_reauth=True, log=True, endpoint_override=None, connect_retries=0, logger=_logger, allow={}, client_name=None, client_version=None, microversion=None, microversion_service_type=None, **kwargs): """Send an HTTP request with the specified characteristics. Wrapper around `requests.Session.request` to handle tasks such as setting headers, JSON encoding/decoding, and error handling. Arguments that are not handled are passed through to the requests library. :param str url: Path or fully qualified URL of HTTP request. If only a path is provided then endpoint_filter must also be provided such that the base URL can be determined. If a fully qualified URL is provided then endpoint_filter will be ignored. :param str method: The http method to use. (e.g. 'GET', 'POST') :param str original_ip: Mark this request as forwarded for this ip. (optional) :param dict headers: Headers to be included in the request. (optional) :param json: Some data to be represented as JSON. (optional) :param str user_agent: A user_agent to use for the request. If present will override one present in headers. (optional) :param int/bool redirect: the maximum number of redirections that can be followed by a request. Either an integer for a specific count or True/False for forever/never. (optional) :param int connect_retries: the maximum number of retries that should be attempted for connection errors. (optional, defaults to 0 - never retry). :param bool authenticated: True if a token should be attached to this request, False if not or None for attach if an auth_plugin is available. (optional, defaults to None) :param dict endpoint_filter: Data to be provided to an auth plugin with which it should be able to determine an endpoint to use for this request. If not provided then URL is expected to be a fully qualified URL. (optional) :param str endpoint_override: The URL to use instead of looking up the endpoint in the auth plugin. This will be ignored if a fully qualified URL is provided but take priority over an endpoint_filter. This string may contain the values ``%(project_id)s`` and ``%(user_id)s`` to have those values replaced by the project_id/user_id of the current authentication. (optional) :param auth: The auth plugin to use when authenticating this request. This will override the plugin that is attached to the session (if any). (optional) :type auth: keystoneauth1.plugin.BaseAuthPlugin :param requests_auth: A requests library auth plugin that cannot be passed via kwarg because the `auth` kwarg collides with our own auth plugins. (optional) :type requests_auth: :py:class:`requests.auth.AuthBase` :param bool raise_exc: If True then raise an appropriate exception for failed HTTP requests. If False then return the request object. (optional, default True) :param bool allow_reauth: Allow fetching a new token and retrying the request on receiving a 401 Unauthorized response. (optional, default True) :param bool log: If True then log the request and response data to the debug log. (optional, default True) :param logger: The logger object to use to log request and responses. If not provided the keystoneauth1.session default logger will be used. :type logger: logging.Logger :param dict allow: Extra filters to pass when discovering API versions. (optional) :param microversion: Microversion to send for this request. microversion can be given as a string or a tuple. (optional) :param str microversion_service_type: The service_type to be sent in the microversion header, if a microversion is given. Defaults to the value of service_type from endpoint_filter if one exists. If endpoint_filter is not provided or does not have a service_type, microversion is given and microversion_service_type is not provided, an exception will be raised. :param kwargs: any other parameter that can be passed to :meth:`requests.Session.request` (such as `headers`). Except: - `data` will be overwritten by the data in the `json` param. - `allow_redirects` is ignored as redirects are handled by the session. :raises keystoneauth1.exceptions.base.ClientException: For connection failure, or to indicate an error response code. :returns: The response to the request. """ headers = kwargs.setdefault('headers', dict()) if microversion: self._set_microversion_headers(headers, microversion, microversion_service_type, endpoint_filter) if authenticated is None: authenticated = bool(auth or self.auth) if authenticated: auth_headers = self.get_auth_headers(auth) if auth_headers is None: msg = 'No valid authentication is available' raise exceptions.AuthorizationFailure(msg) headers.update(auth_headers) if osprofiler_web: headers.update(osprofiler_web.get_trace_id_headers()) # if we are passed a fully qualified URL and an endpoint_filter we # should ignore the filter. This will make it easier for clients who # want to overrule the default endpoint_filter data added to all client # requests. We check fully qualified here by the presence of a host. if not urllib.parse.urlparse(url).netloc: base_url = None if endpoint_override: base_url = endpoint_override % _StringFormatter(self, auth) elif endpoint_filter: base_url = self.get_endpoint(auth, allow=allow, **endpoint_filter) if not base_url: raise exceptions.EndpointNotFound() url = '%s/%s' % (base_url.rstrip('/'), url.lstrip('/')) if self.cert: kwargs.setdefault('cert', self.cert) if self.timeout is not None: kwargs.setdefault('timeout', self.timeout) if user_agent: headers['User-Agent'] = user_agent elif self.user_agent: user_agent = headers.setdefault('User-Agent', self.user_agent) else: # Per RFC 7231 Section 5.5.3, identifiers in a user-agent should be # ordered by decreasing significance. If a user sets their product # that value will be used. Otherwise we attempt to derive a useful # product value. The value will be prepended it to the KSA version, # requests version, and then the Python version. agent = [] if self.app_name and self.app_version: agent.append('%s/%s' % (self.app_name, self.app_version)) elif self.app_name: agent.append(self.app_name) for additional in self.additional_user_agent: agent.append('%s/%s' % additional) if client_name and client_version: agent.append('%s/%s' % (client_name, client_version)) elif client_name: agent.append(client_name) if not agent: # NOTE(jamielennox): determine_user_agent will return an empty # string on failure so checking for None will ensure it is only # called once even on failure. if self._determined_user_agent is None: self._determined_user_agent = _determine_user_agent() if self._determined_user_agent: agent.append(self._determined_user_agent) agent.append(DEFAULT_USER_AGENT) user_agent = headers.setdefault('User-Agent', ' '.join(agent)) if self.original_ip: headers.setdefault('Forwarded', 'for=%s;by=%s' % (self.original_ip, user_agent)) if json is not None: headers.setdefault('Content-Type', 'application/json') kwargs['data'] = self._json.encode(json) for k, v in self.additional_headers.items(): headers.setdefault(k, v) kwargs.setdefault('verify', self.verify) if requests_auth: kwargs['auth'] = requests_auth # Query parameters that are included in the url string will # be logged properly, but those sent in the `params` parameter # (which the requests library handles) need to be explicitly # picked out so they can be included in the URL that gets loggged. query_params = kwargs.get('params', dict()) if log: self._http_log_request(url, method=method, data=kwargs.get('data'), headers=headers, query_params=query_params, logger=logger) # Force disable requests redirect handling. We will manage this below. kwargs['allow_redirects'] = False if redirect is None: redirect = self.redirect send = functools.partial(self._send_request, url, method, redirect, log, logger, connect_retries) try: connection_params = self.get_auth_connection_params(auth=auth) except exceptions.MissingAuthPlugin: # NOTE(jamielennox): If we've gotten this far without an auth # plugin then we should be happy with allowing no additional # connection params. This will be the typical case for plugins # anyway. pass else: if connection_params: kwargs.update(connection_params) resp = send(**kwargs) # log callee and caller request-id for each api call if log: # service_name should be fetched from endpoint_filter if it is not # present then use service_type as service_name. service_name = None if endpoint_filter: service_name = endpoint_filter.get('service_name') if not service_name: service_name = endpoint_filter.get('service_type') # Nova uses 'x-compute-request-id' and other services like # Glance, Cinder etc are using 'x-openstack-request-id' to store # request-id in the header request_id = (resp.headers.get('x-openstack-request-id') or resp.headers.get('x-compute-request-id')) if request_id: logger.debug( '%(method)s call to %(service_name)s for ' '%(url)s used request id ' '%(response_request_id)s', { 'method': resp.request.method, 'service_name': service_name, 'url': resp.url, 'response_request_id': request_id }) # handle getting a 401 Unauthorized response by invalidating the plugin # and then retrying the request. This is only tried once. if resp.status_code == 401 and authenticated and allow_reauth: if self.invalidate(auth): auth_headers = self.get_auth_headers(auth) if auth_headers is not None: headers.update(auth_headers) resp = send(**kwargs) if raise_exc and resp.status_code >= 400: logger.debug('Request returned failure status: %s', resp.status_code) raise exceptions.from_response(resp, method, url) return resp
def test_get_trace_id_headers_no_hmac(self): profiler.init(None, base_id="y", parent_id="z") headers = web.get_trace_id_headers() self.assertEqual(headers, {})
def request(self, url, method, json=None, original_ip=None, user_agent=None, redirect=None, authenticated=None, endpoint_filter=None, auth=None, requests_auth=None, raise_exc=True, allow_reauth=True, log=True, endpoint_override=None, connect_retries=0, logger=_logger, **kwargs): """Send an HTTP request with the specified characteristics. Wrapper around `requests.Session.request` to handle tasks such as setting headers, JSON encoding/decoding, and error handling. Arguments that are not handled are passed through to the requests library. :param str url: Path or fully qualified URL of HTTP request. If only a path is provided then endpoint_filter must also be provided such that the base URL can be determined. If a fully qualified URL is provided then endpoint_filter will be ignored. :param str method: The http method to use. (e.g. 'GET', 'POST') :param str original_ip: Mark this request as forwarded for this ip. (optional) :param dict headers: Headers to be included in the request. (optional) :param json: Some data to be represented as JSON. (optional) :param str user_agent: A user_agent to use for the request. If present will override one present in headers. (optional) :param int/bool redirect: the maximum number of redirections that can be followed by a request. Either an integer for a specific count or True/False for forever/never. (optional) :param int connect_retries: the maximum number of retries that should be attempted for connection errors. (optional, defaults to 0 - never retry). :param bool authenticated: True if a token should be attached to this request, False if not or None for attach if an auth_plugin is available. (optional, defaults to None) :param dict endpoint_filter: Data to be provided to an auth plugin with which it should be able to determine an endpoint to use for this request. If not provided then URL is expected to be a fully qualified URL. (optional) :param str endpoint_override: The URL to use instead of looking up the endpoint in the auth plugin. This will be ignored if a fully qualified URL is provided but take priority over an endpoint_filter. This string may contain the values ``%(project_id)s`` and ``%(user_id)s`` to have those values replaced by the project_id/user_id of the current authentication. (optional) :param auth: The auth plugin to use when authenticating this request. This will override the plugin that is attached to the session (if any). (optional) :type auth: keystoneauth1.plugin.BaseAuthPlugin :param requests_auth: A requests library auth plugin that cannot be passed via kwarg because the `auth` kwarg collides with our own auth plugins. (optional) :type requests_auth: :py:class:`requests.auth.AuthBase` :param bool raise_exc: If True then raise an appropriate exception for failed HTTP requests. If False then return the request object. (optional, default True) :param bool allow_reauth: Allow fetching a new token and retrying the request on receiving a 401 Unauthorized response. (optional, default True) :param bool log: If True then log the request and response data to the debug log. (optional, default True) :param logger: The logger object to use to log request and responses. If not provided the keystoneauth1.session default logger will be used. :type logger: logging.Logger :param kwargs: any other parameter that can be passed to :meth:`requests.Session.request` (such as `headers`). Except: - `data` will be overwritten by the data in the `json` param. - `allow_redirects` is ignored as redirects are handled by the session. :raises keystoneauth1.exceptions.base.ClientException: For connection failure, or to indicate an error response code. :returns: The response to the request. """ headers = kwargs.setdefault('headers', dict()) if authenticated is None: authenticated = bool(auth or self.auth) if authenticated: auth_headers = self.get_auth_headers(auth) if auth_headers is None: msg = 'No valid authentication is available' raise exceptions.AuthorizationFailure(msg) headers.update(auth_headers) if osprofiler_web: headers.update(osprofiler_web.get_trace_id_headers()) # if we are passed a fully qualified URL and an endpoint_filter we # should ignore the filter. This will make it easier for clients who # want to overrule the default endpoint_filter data added to all client # requests. We check fully qualified here by the presence of a host. if not urllib.parse.urlparse(url).netloc: base_url = None if endpoint_override: base_url = endpoint_override % _StringFormatter(self, auth) elif endpoint_filter: base_url = self.get_endpoint(auth, **endpoint_filter) if not base_url: raise exceptions.EndpointNotFound() url = '%s/%s' % (base_url.rstrip('/'), url.lstrip('/')) if self.cert: kwargs.setdefault('cert', self.cert) if self.timeout is not None: kwargs.setdefault('timeout', self.timeout) if user_agent: headers['User-Agent'] = user_agent elif self.user_agent: user_agent = headers.setdefault('User-Agent', self.user_agent) else: user_agent = headers.setdefault('User-Agent', DEFAULT_USER_AGENT) if self.original_ip: headers.setdefault('Forwarded', 'for=%s;by=%s' % (self.original_ip, user_agent)) if json is not None: headers['Content-Type'] = 'application/json' kwargs['data'] = self._json.encode(json) kwargs.setdefault('verify', self.verify) if requests_auth: kwargs['auth'] = requests_auth if log: self._http_log_request(url, method=method, data=kwargs.get('data'), headers=headers, logger=logger) # Force disable requests redirect handling. We will manage this below. kwargs['allow_redirects'] = False if redirect is None: redirect = self.redirect send = functools.partial(self._send_request, url, method, redirect, log, logger, connect_retries) try: connection_params = self.get_auth_connection_params(auth=auth) except exceptions.MissingAuthPlugin: # NOTE(jamielennox): If we've gotten this far without an auth # plugin then we should be happy with allowing no additional # connection params. This will be the typical case for plugins # anyway. pass else: if connection_params: kwargs.update(connection_params) resp = send(**kwargs) # handle getting a 401 Unauthorized response by invalidating the plugin # and then retrying the request. This is only tried once. if resp.status_code == 401 and authenticated and allow_reauth: if self.invalidate(auth): auth_headers = self.get_auth_headers(auth) if auth_headers is not None: headers.update(auth_headers) resp = send(**kwargs) if raise_exc and resp.status_code >= 400: logger.debug('Request returned failure status: %s', resp.status_code) raise exceptions.from_response(resp, method, url) return resp