class OpenStackRateLimitMiddleware(object):
"""
OpenStack Rate Limit Middleware enforces configurable rate limits.
Per combination of:
service ( compute, identity, object-store, .. )
scope ( initiator|target project uid, initiator host address )
target_type_uri ( service/compute/servers, service/storage/block/volumes,.. )
action ( create, read, update, delete, authenticate, .. )
"""
def __init__(self, app, **conf):
self.app = app
# Configuration via paste.ini.
self.__conf = conf
self.logger = log.Logger(conf.get('log_name', __name__))
# StatsD is used to emit metrics.
statsd_host = self.__conf.get('statsd_host', '127.0.0.1')
statsd_port = common.to_int(self.__conf.get('statsd_port', 9125))
statsd_prefix = self.__conf.get('statsd_prefix',
common.Constants.metric_prefix)
# Init StatsD client.
self.metricsClient = DogStatsd(
host=os.getenv('STATSD_HOST', statsd_host),
port=int(os.getenv('STATSD_PORT', statsd_port)),
namespace=os.getenv('STATSD_PREFIX', statsd_prefix))
# Get backend configuration.
# Backend is used to store count of requests.
self.backend_host = self.__conf.get('backend_host', '127.0.0.1')
self.backend_port = common.to_int(self.__conf.get('backend_port'),
6379)
self.logger.debug("using backend '{0}' on '{1}:{2}'".format(
'redis', self.backend_host, self.backend_port))
backend_timeout_seconds = common.to_int(
self.__conf.get('backend_timeout_seconds'), 20)
backend_max_connections = common.to_int(
self.__conf.get('backend_max_connections'), 100)
# Load configuration file.
self.config = {}
config_file = self.__conf.get('config_file', None)
if config_file:
try:
self.config = common.load_config(config_file)
except errors.ConfigError as e:
self.logger.warning("error loading configuration: {0}".format(
str(e)))
self.service_type = self.__conf.get('service_type', None)
# This is required to trim the prefix from the target_type_uri.
# Example:
# service_type = identity
# cadf_service_name = data/security
# target_type_uri = data/security/auth/tokens -> auth/tokens
self.cadf_service_name = self.__conf.get('cadf_service_name', None)
if common.is_none_or_unknown(self.cadf_service_name):
self.cadf_service_name = common.CADF_SERVICE_TYPE_PREFIX_MAP.get(
self.service_type, None)
# Use configured parameters or ensure defaults.
max_sleep_time_seconds = common.to_int(
self.__conf.get(common.Constants.max_sleep_time_seconds), 20)
log_sleep_time_seconds = common.to_int(
self.__conf.get(common.Constants.log_sleep_time_seconds), 10)
# Setup ratelimit and blacklist response.
self._setup_response()
# White-/blacklist can contain project, domain, user ids or the client ip address.
# Don't apply rate limits to localhost.
default_whitelist = ['127.0.0.1', 'localhost']
config_whitelist = self.config.get('whitelist', [])
self.whitelist = default_whitelist + config_whitelist
self.whitelist_users = self.config.get('whitelist_users', [])
self.blacklist = self.config.get('blacklist', [])
self.blacklist_users = self.config.get('blacklist_users', [])
# Mapping of potentially multiple CADF actions to one action.
self.rate_limit_groups = self.config.get('groups', {})
# Configurable scope in which a rate limit is applied. Defaults to initiator project id.
# Rate limits are applied based on the tuple of (rate_limit_by, action, target_type_uri).
self.rate_limit_by = self.__conf.get(
'rate_limit_by', common.Constants.initiator_project_id)
# Accuracy of the request timestamps used. Defaults to nanosecond accuracy.
clock_accuracy = int(
1 / units.Units.parse(self.__conf.get('clock_accuracy', '1ns')))
self.backend = rate_limit_backend.RedisBackend(
host=self.backend_host,
port=self.backend_port,
rate_limit_response=self.ratelimit_response,
max_sleep_time_seconds=max_sleep_time_seconds,
log_sleep_time_seconds=log_sleep_time_seconds,
timeout_seconds=backend_timeout_seconds,
max_connections=backend_max_connections,
clock_accuracy=clock_accuracy,
)
# Test if the backend is ready.
is_available, msg = self.backend.is_available()
if not is_available:
self.logger.warning(
"rate limit not possible. the backend is not available: {0}".
format(msg))
# Provider for rate limits. Defaults to configuration file.
# Also supports Limes.
configuration_ratelimit_provider = provider.ConfigurationRateLimitProvider(
service_type=self.service_type)
# Force load of rate limits from configuration file.
configuration_ratelimit_provider.read_rate_limits_from_config(
config_file)
self.ratelimit_provider = configuration_ratelimit_provider
# If limes is enabled and we want to rate limit by initiator|target project id,
# Set LimesRateLimitProvider as the provider for rate limits.
limes_enabled = self.__conf.get('limes_enabled', False)
if limes_enabled:
self.__setup_limes_ratelimit_provider()
self.logger.info("OpenStack Rate Limit Middleware ready for requests.")
def _setup_response(self):
"""Setup configurable RateLimitExceededResponse and BlacklistResponse."""
# Default responses.
ratelimit_response = response.RateLimitExceededResponse()
blacklist_response = response.BlacklistResponse()
# Overwrite default responses if custom ones are configured.
try:
ratelimit_response_config = self.config.get(
common.Constants.ratelimit_response)
if ratelimit_response_config:
status, status_code, headers, body, json_body = \
response.response_parameters_from_config(ratelimit_response_config)
# Only create custom response if all parameters are given.
if status and status_code and (body or json_body):
ratelimit_response = response.RateLimitExceededResponse(
status=status,
status_code=status_code,
headerlist=headers,
body=body,
json_body=json_body)
blacklist_response_config = self.config.get(
common.Constants.blacklist_response)
if blacklist_response_config:
status, status_code, headers, body, json_body = \
response.response_parameters_from_config(blacklist_response_config)
# Only create custom response if all parameters are given.
if status and status_code and (body or json_body):
blacklist_response = response.BlacklistResponse(
status=status,
status_code=status_code,
headerlist=headers,
body=body,
json_body=json_body)
except Exception as e:
self.logger.debug(
"error configuring custom responses. falling back to defaults: {0}"
.format(str(e)))
finally:
self.ratelimit_response = ratelimit_response
self.blacklist_response = blacklist_response
def __setup_limes_ratelimit_provider(self):
"""Setup Limes as provider for rate limits. If not successful fallback to configuration file."""
try:
limes_ratelimit_provider = provider.LimesRateLimitProvider(
service_type=self.service_type,
redis_host=self.backend_host,
redis_port=self.backend_port,
refresh_interval_seconds=self.__conf.get(
common.Constants.limes_refresh_interval_seconds, 300),
limes_api_uri=self.__conf.get(common.Constants.limes_api_uri),
auth_url=self.__conf.get('identity_auth_url'),
username=self.__conf.get('username'),
user_domain_name=self.__conf.get('user_domain_name'),
password=self.__conf.get('password'),
domain_name=self.__conf.get('domain_name'))
self.ratelimit_provider = limes_ratelimit_provider
except Exception as e:
self.logger.debug(
"failed to setup limes rate limit provider: {0}".format(
str(e)))
@classmethod
def factory(cls, global_config, **local_config):
conf = global_config.copy()
conf.update(local_config)
def limiter(app):
return cls(app, **conf)
return limiter
def _rate_limit(self, scope, action, target_type_uri, **kwargs):
"""
Check the whitelist, blacklist, global and local ratelimits.
:param scope: the scope of the request
:param action: the action of the request
:param target_type_uri: the target type URI of the response
:return: None or BlacklistResponse or RateLimitResponse
"""
# Labels used for all metrics.
metric_labels = [
'service:{0}'.format(self.service_type),
'service_name:{0}'.format(self.cadf_service_name),
'action:{0}'.format(action),
'{0}:{1}'.format(self.rate_limit_by, scope),
'target_type_uri:{0}'.format(target_type_uri)
]
global_metric_labels = metric_labels + ['level:global']
local_metric_labels = metric_labels + ['level:local']
# Check whether a set of CADF actions are accounted together.
new_action = self.get_action_from_rate_limit_groups(action)
if not common.is_none_or_unknown(new_action):
action = new_action
metric_labels.append('action_group:{0}'.format(action))
# Get CADF service name and trim from target_type_uri.
trimmed_target_type_uri = target_type_uri
if not common.is_none_or_unknown(self.cadf_service_name):
trimmed_target_type_uri = self._trim_cadf_service_prefix_from_target_type_uri(
self.cadf_service_name, target_type_uri)
username = kwargs.get('username', None)
# If we have the username: Check whether the user is white- or blacklisted.
if username:
metric_labels.append('initiator_user_name:{}'.format(username))
if self.is_user_whitelisted(username):
self.logger.debug(
"user {0} is whitelisted. skipping rate limit".format(
username))
self.metricsClient.increment(
common.Constants.metric_requests_whitelisted_total,
tags=metric_labels)
return None
if self.is_user_blacklisted(username):
self.logger.debug(
"user {0} is blacklisted. returning BlacklistResponse".
format(username))
self.metricsClient.increment(
common.Constants.metric_requests_blacklisted_total,
tags=metric_labels)
return self.blacklist_response
# The key of the scope in the format $domainName/projectName.
scope_name_key = kwargs.get('scope_name_key', None)
if scope_name_key:
metric_labels.append(
'initiator_project_name:{}'.format(scope_name_key))
# Check whitelist. If scope is whitelisted break here and don't apply any rate limits.
if self.is_scope_whitelisted(scope) or self.is_scope_whitelisted(
scope_name_key):
self.logger.debug(
"scope {0} (key: {1}) is whitelisted. skipping rate limit".
format(scope, scope_name_key))
self.metricsClient.increment(
common.Constants.metric_requests_whitelisted_total,
tags=metric_labels)
return None
# Check blacklist. If scope is blacklisted return BlacklistResponse.
if self.is_scope_blacklisted(scope) or self.is_scope_blacklisted(
scope_name_key):
self.logger.debug(
"scope {0} (key: {1}) is blacklisted. returning BlacklistResponse"
.format(scope, scope_name_key))
self.metricsClient.increment(
common.Constants.metric_requests_blacklisted_total,
tags=metric_labels)
return self.blacklist_response
# Get global rate limits from the provider.
global_rate_limit = self.ratelimit_provider.get_global_rate_limits(
action, trimmed_target_type_uri)
# Don't rate limit if limit=-1 or unknown.
if not common.is_unlimited(global_rate_limit):
self.logger.debug(
"global rate limit configured for request with action '{0}', target type URI '{1}': '{2}'"
.format(action, target_type_uri, global_rate_limit))
# Check global rate limits.
# Global rate limits enforce a backend protection by counting all requests independent of their scope.
rate_limit_response = self.backend.rate_limit(
scope=None,
action=action,
target_type_uri=trimmed_target_type_uri,
max_rate_string=global_rate_limit)
if rate_limit_response:
self.metricsClient.increment(
common.Constants.metric_requests_ratelimit_total,
tags=global_metric_labels)
return rate_limit_response
# Get local (for a certain scope) rate limits from provider.
local_rate_limit = self.ratelimit_provider.get_local_rate_limits(
scope, action, trimmed_target_type_uri)
# Don't rate limit for rate_limit=-1 or if unknown.
if not common.is_unlimited(local_rate_limit):
self.logger.debug(
"local rate limit configured for request with action '{0}', target type URI '{1}', scope '{2}': '{3}'"
.format(action, target_type_uri, scope, local_rate_limit))
# Check local (for a specific scope) rate limits.
rate_limit_response = self.backend.rate_limit(
scope=scope,
action=action,
target_type_uri=trimmed_target_type_uri,
max_rate_string=local_rate_limit)
if rate_limit_response:
self.metricsClient.increment(
common.Constants.metric_requests_ratelimit_total,
tags=local_metric_labels)
return rate_limit_response
return None
def __call__(self, environ, start_response):
"""
WSGI entry point. Wraps environ in webob.Request.
:param environ: the WSGI environment dict
:param start_response: WSGI callable
"""
# Save the app's response so it can be returned easily.
resp = self.app
try:
self.metricsClient.open_buffer()
# If the service type and/or service name is not configured,
# attempt to extract watcher classification from environ and set it.
self._set_service_type_and_name(environ)
# Get openstack-watcher-middleware classification from requests environ.
scope, action, target_type_uri = self.get_scope_action_target_type_uri_from_environ(
environ)
# Don't rate limit if any of scope, action, target type URI cannot be determined.
if common.is_none_or_unknown(scope) or \
common.is_none_or_unknown(action) or \
common.is_none_or_unknown(target_type_uri):
path = str(environ.get('PATH_INFO', common.Constants.unknown))
method = str(
environ.get('REQUEST_METHOD', common.Constants.unknown))
self.logger.debug(
"unknown request: action: {0}, target_type_uri: {1}, scope: {2}, method: {3}, path: {4}"
.format(action, target_type_uri, scope, method, path))
self.metricsClient.increment(
common.Constants.metric_requests_unknown_classification,
tags=[
'service:{0}'.format(self.service_type),
'service_name:{0}'.format(self.cadf_service_name),
])
return
# Returns a RateLimitResponse or BlacklistResponse or None, in which case the original response is returned.
rate_limit_response = self._rate_limit(
scope=scope,
action=action,
target_type_uri=target_type_uri,
scope_name_key=self._get_scope_name_key_from_environ(environ),
username=self._get_username_from_environ(environ),
)
if rate_limit_response:
rate_limit_response.set_environ(environ)
resp = rate_limit_response
except Exception as e:
self.metricsClient.increment(common.Constants.metric_errors_total)
self.logger.debug("checking rate limits failed with: {0}".format(
str(e)))
finally:
self.metricsClient.close_buffer()
return resp(environ, start_response)
def is_scope_blacklisted(self, key_to_check):
"""
Check whether a scope (user_id, project_id or client ip) is blacklisted.
:param key_to_check: the user, project uid or client ip
:return: bool whether the key is blacklisted
"""
for entry in self.blacklist:
if entry == key_to_check:
return True
return False
def is_user_blacklisted(self, user_to_check):
"""
Check whether a user is blacklisted.
:param user_to_check: the name of the user to check
:return: bool whether user is blacklisted
"""
for u in self.blacklist_users:
if str(u).lower() == str(user_to_check).lower():
return True
return False
def is_scope_whitelisted(self, key_to_check):
"""
Check whether a scope (user_id, project_id or client ip) is whitelisted.
:param key_to_check: the user, project uid or client ip
:return: bool whether the key is whitelisted
"""
for entry in self.whitelist:
if entry == key_to_check:
return True
return False
def is_user_whitelisted(self, user_to_check):
"""
Check whether a user is whitelisted.
:param user_to_check: the name of the user to check
:return: bool whether user is whitelisted
"""
for u in self.whitelist_users:
if str(u).lower() == str(user_to_check).lower():
return True
return False
def get_scope_action_target_type_uri_from_environ(self, environ):
"""
Get the scope, action, target type URI from the request environ.
:param environ: the request environ
:return: tuple of scope, action, target type URI
"""
action = target_type_uri = scope = None
try:
# Get the CADF action.
env_action = environ.get('WATCHER.ACTION')
if not common.is_none_or_unknown(env_action):
action = env_action
# Get the target type URI.
env_target_type_uri = environ.get('WATCHER.TARGET_TYPE_URI')
if not common.is_none_or_unknown(env_target_type_uri):
target_type_uri = env_target_type_uri
# Get scope from request environment, which might be an initiator.project_id, target.project_id, etc. .
env_scope = self._get_scope_from_environ(environ)
if not common.is_none_or_unknown(env_scope):
scope = env_scope
except Exception as e:
self.logger.debug(
"error while getting scope, action, target type URI from environ: {0}"
.format(str(e)))
finally:
self.logger.debug(
'got WATCHER.* attributes from environ: action: {0}, target_type_uri: {1}, scope: {2}'
.format(action, target_type_uri, scope))
return scope, action, target_type_uri
def _get_scope_from_environ(self, environ):
"""
Get the scope from the requests environ.
The scope is configurable and may be the target|initiator project uid or the initiator host address.
Default to initiator project ID.
:param environ: the requests environ
:return: the scope
"""
scope = None
if self.rate_limit_by == common.Constants.target_project_id:
env_scope = environ.get('WATCHER.TARGET_PROJECT_ID', None)
elif self.rate_limit_by == common.Constants.initiator_host_address:
env_scope = environ.get('WATCHER.INITIATOR_HOST_ADDRESS', None)
else:
env_scope = environ.get('WATCHER.INITIATOR_PROJECT_ID', None)
# Ensure the scope is not 'unknown'.
if not common.is_none_or_unknown(env_scope):
scope = env_scope
return scope
def _get_scope_name_key_from_environ(self, environ):
"""
Attempt to build the key '$domainName/$projectName' from the WATCHER attributes found in the request environ.
:param environ: the request environ
:return: the key or None
"""
_domain_name = environ.get('WATCHER.INITIATOR_DOMAIN_NAME', None)
_project_domain_name = environ.get(
'WATCHER.INITIATOR_PROJECT_DOMAIN_NAME', None)
project_name = environ.get('WATCHER.INITIATOR_PROJECT_NAME', None)
domain_name = _project_domain_name or _domain_name
if common.is_none_or_unknown(
project_name) or common.is_none_or_unknown(domain_name):
return None
return '{0}/{1}'.format(domain_name, project_name)
def _get_username_from_environ(self, environ):
"""
Attempt to get username from WATCHER attributes found in request environ.
:param environ: the request environ
:return: the username or None
"""
username = environ.get('WATCHER.INITIATOR_USER_NAME', None)
if common.is_none_or_unknown(username):
return None
return username
def _set_service_type_and_name(self, environ):
"""
Set the service type and name according to the watchers classification passed in the request WSGI environ.
Used if nothing was configured.
:param environ: the request WSGI environment
"""
# Get service type from request environ.
if common.is_none_or_unknown(self.service_type):
svc_type = environ.get('WATCHER.SERVICE_TYPE')
if not common.is_none_or_unknown(svc_type):
self.service_type = svc_type
self.ratelimit_provider.service_type = self.service_type
# set service name from environ
if common.is_none_or_unknown(self.cadf_service_name):
svc_name = environ.get('WATCHER.CADF_SERVICE_NAME')
if not common.is_none_or_unknown(svc_name):
self.cadf_service_name = svc_name
self.ratelimit_provider.cadf_service_name = self.cadf_service_name
def _trim_cadf_service_prefix_from_target_type_uri(self, prefix,
target_type_uri):
"""
Get cadf service name and trim from target_type_uri.
Example:
target_type_uri: service/storage/object/account/container/object
cadf_service_name: service/storage/object
=> trimmed_target_type_uri: account/container/object
:param prefix: the cadf service name prefixing the target_type_uri
:param target_type_uri: the target_type_uri with the prefix
:return: target_type_uri without prefix
"""
target_type_uri_without_prefix = target_type_uri
try:
without_prefix = target_type_uri.split(prefix)
if len(without_prefix) != 2:
raise IndexError
target_type_uri_without_prefix = without_prefix[-1].lstrip('/')
except IndexError as e:
self.logger.warning(
"rate limiting might not be possible. cannot trim prefix '{0}' from target_type_uri '{1}': {2}"
.format(prefix, target_type_uri, str(e)))
finally:
return target_type_uri_without_prefix
def get_action_from_rate_limit_groups(self, action):
"""
Multiple CADF actions can be grouped and accounted as one entity.
:param action: the original CADF action
:return: the original action or action as per grouping
"""
for group in self.rate_limit_groups:
if action in self.rate_limit_groups[group]:
return group
return action
class OpenStackWatcherMiddleware(object):
"""
OpenStack Watcher Middleware
Watches OpenStack traffic and classifies according to CADF standard
"""
def __init__(self, app, config, logger=logging.getLogger(__name__)):
self.logger = logger
self.app = app
self.wsgi_config = config
self.watcher_config = {}
self.cadf_service_name = self.wsgi_config.get('cadf_service_name',
None)
self.service_type = self.wsgi_config.get('service_type',
taxonomy.UNKNOWN)
# get the project uid from the request path or from the token (default)
self.is_project_id_from_path = common.string_to_bool(
self.wsgi_config.get('target_project_id_from_path', 'False'))
# get the project id from the service catalog (see documentation on keystone auth_token middleware)
self.is_project_id_from_service_catalog = common.string_to_bool(
self.wsgi_config.get('target_project_id_from_service_catalog',
'False'))
# whether to include the target project id in the metrics
self.is_include_target_project_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_target_project_id_in_metric',
'True'))
# whether to include the target domain id in the metrics
self.is_include_target_domain_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_target_domain_id_in_metric', 'True'))
# whether to include the initiator user id in the metrics
self.is_include_initiator_user_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_initiator_user_id_in_metric',
'False'))
config_file_path = config.get('config_file', None)
if config_file_path:
try:
self.watcher_config = load_config(config_file_path)
except errors.ConfigError as e:
self.logger.debug("custom actions not available: %s", str(e))
custom_action_config = self.watcher_config.get('custom_actions', {})
path_keywords = self.watcher_config.get('path_keywords', {})
keyword_exclusions = self.watcher_config.get('keyword_exclusions', {})
regex_mapping = self.watcher_config.get('regex_path_mapping', {})
# init the strategy used to determine the target type uri
strat = STRATEGIES.get(self.service_type, strategies.BaseCADFStrategy)
# set custom prefix to target type URI or use defaults
target_type_uri_prefix = common.SERVICE_TYPE_CADF_PREFIX_MAP.get(
self.service_type, 'service/{0}'.format(self.service_type))
if self.cadf_service_name:
target_type_uri_prefix = self.cadf_service_name
strategy = strat(target_type_uri_prefix=target_type_uri_prefix,
path_keywords=path_keywords,
keyword_exclusions=keyword_exclusions,
custom_action_config=custom_action_config,
regex_mapping=regex_mapping)
self.strategy = strategy
self.metric_client = DogStatsd(
host=self.wsgi_config.get("statsd_host", "127.0.0.1"),
port=int(self.wsgi_config.get("statsd_port", 9125)),
namespace=self.wsgi_config.get("statsd_namespace",
"openstack_watcher"))
@classmethod
def factory(cls, global_config, **local_config):
conf = global_config.copy()
conf.update(local_config)
def watcher(app):
return cls(app, conf)
return watcher
def __call__(self, environ, start_response):
"""
WSGI entry point. Wraps environ in webob.Request
:param environ: the WSGI environment dict
:param start_response: WSGI callable
"""
# capture start timestamp
start = time.time()
req = Request(environ)
# determine initiator based on token context
initiator_project_id = self.get_safe_from_environ(
environ, 'HTTP_X_PROJECT_ID')
initiator_project_name = self.get_safe_from_environ(
environ, 'HTTP_X_PROJECT_NAME')
initiator_project_domain_id = self.get_safe_from_environ(
environ, 'HTTP_X_PROJECT_DOMAIN_ID')
initiator_project_domain_name = self.get_safe_from_environ(
environ, 'HTTP_X_PROJECT_DOMAIN_NAME')
initiator_domain_id = self.get_safe_from_environ(
environ, 'HTTP_X_DOMAIN_ID')
initiator_domain_name = self.get_safe_from_environ(
environ, 'HTTP_X_DOMAIN_NAME')
initiator_user_id = self.get_safe_from_environ(environ,
'HTTP_X_USER_ID')
initiator_user_name = self.get_safe_from_environ(
environ, 'HTTP_X_USER_NAME')
initiator_user_domain_id = self.get_safe_from_environ(
environ, 'HTTP_X_USER_DOMAIN_ID')
initiator_user_domain_name = self.get_safe_from_environ(
environ, 'HTTP_X_USER_DOMAIN_NAME')
initiator_host_address = req.client_addr or taxonomy.UNKNOWN
# determine target based on request path or keystone.token_info
target_project_id = taxonomy.UNKNOWN
if self.is_project_id_from_path:
target_project_id = self.get_target_project_uid_from_path(req.path)
elif self.is_project_id_from_service_catalog:
target_project_id = self.get_target_project_id_from_keystone_token_info(
environ.get('keystone.token_info'))
# default target_project_id to initiator_project_id if still unknown
if not target_project_id or target_project_id == taxonomy.UNKNOWN:
target_project_id = initiator_project_id
# determine target.type_uri for request
target_type_uri = self.determine_target_type_uri(req)
# determine cadf_action for request. consider custom action config.
cadf_action = self.determine_cadf_action(req, target_type_uri)
# if authentication request consider project, domain and user in body
if self.service_type == 'identity' and cadf_action == taxonomy.ACTION_AUTHENTICATE:
initiator_project_id, initiator_domain_id, initiator_user_id = \
self.get_project_domain_and_user_id_from_keystone_authentication_request(req)
# set environ for initiator
environ['WATCHER.INITIATOR_PROJECT_ID'] = initiator_project_id
environ['WATCHER.INITIATOR_PROJECT_NAME'] = initiator_project_name
environ[
'WATCHER.INITIATOR_PROJECT_DOMAIN_ID'] = initiator_project_domain_id
environ[
'WATCHER.INITIATOR_PROJECT_DOMAIN_NAME'] = initiator_project_domain_name
environ['WATCHER.INITIATOR_DOMAIN_ID'] = initiator_domain_id
environ['WATCHER.INITIATOR_DOMAIN_NAME'] = initiator_domain_name
environ['WATCHER.INITIATOR_USER_ID'] = initiator_user_id
environ['WATCHER.INITIATOR_USER_NAME'] = initiator_user_name
environ['WATCHER.INITIATOR_USER_DOMAIN_ID'] = initiator_user_domain_id
environ[
'WATCHER.INITIATOR_USER_DOMAIN_NAME'] = initiator_user_domain_name
environ['WATCHER.INITIATOR_HOST_ADDRESS'] = initiator_host_address
# set environ for target
environ['WATCHER.TARGET_PROJECT_ID'] = target_project_id
environ['WATCHER.TARGET_TYPE_URI'] = target_type_uri
# general cadf attributes
environ['WATCHER.ACTION'] = cadf_action
environ['WATCHER.SERVICE_TYPE'] = self.service_type
environ[
'WATCHER.CADF_SERVICE_NAME'] = self.strategy.get_cadf_service_name(
)
# labels applied to all metrics emitted by this middleware
labels = [
"service_name:{0}".format(self.strategy.get_cadf_service_name()),
"service:{0}".format(self.service_type),
"action:{0}".format(cadf_action),
"target_type_uri:{0}".format(target_type_uri),
]
# additional labels not needed in all metrics
detail_labels = [
"initiator_project_id:{0}".format(initiator_project_id),
"initiator_domain_id:{0}".format(initiator_domain_id),
]
detail_labels = labels + detail_labels
# include the target project id in metric
if self.is_include_target_project_id_in_metric:
detail_labels.append(
"target_project_id:{0}".format(target_project_id))
# include initiator user id
if self.is_include_initiator_user_id_in_metric:
detail_labels.append(
"initiator_user_id:{0}".format(initiator_user_id))
# if swift request: determine target.container_id based on request path
if common.is_swift_request(
req.path) or self.service_type == 'object-store':
_, target_container_id = self.get_target_account_container_id_from_request(
req)
environ['WATCHER.TARGET_CONTAINER_ID'] = target_container_id
self.logger.debug(
'got request with initiator_project_id: {0}, initiator_domain_id: {1}, initiator_user_id: {2}, '
'target_project_id: {3}, action: {4}, target_type_uri: {5}'.format(
initiator_project_id, initiator_domain_id, initiator_user_id,
target_project_id, cadf_action, target_type_uri))
# capture the response status
response_wrapper = {}
try:
def _start_response_wrapper(status, headers, exc_info=None):
response_wrapper.update(status=status,
headers=headers,
exc_info=exc_info)
return start_response(status, headers, exc_info)
return self.app(environ, _start_response_wrapper)
finally:
try:
self.metric_client.open_buffer()
status = response_wrapper.get('status')
if status:
status_code = status.split()[0]
else:
status_code = taxonomy.UNKNOWN
labels.append("status:{0}".format(status_code))
detail_labels.append("status:{0}".format(status_code))
self.metric_client.timing(
'api_requests_duration_seconds',
int(round(1000 * (time.time() - start))),
tags=labels)
self.metric_client.increment('api_requests_total',
tags=detail_labels)
except Exception as e:
self.logger.debug("failed to submit metrics for %s: %s" %
(str(labels), str(e)))
finally:
self.metric_client.close_buffer()
def get_safe_from_environ(self, environ, key, default=taxonomy.UNKNOWN):
"""
get value for a key from the environ dict ensuring it's never None or an empty string
:param environ: the request environ
:param key: the key in the environ dictionary
:param default: return value if key not found
:return: the value to the key or default
"""
val = default
try:
v = environ.get(key, default)
if v and v != "":
val = v
except Exception as e:
self.logger.debug("error getting '{0}' from environ: {1}".format(
key, e))
finally:
return val
def get_target_project_uid_from_path(self, path):
"""
get the project uid from the path, which should look like
../v1.2/<project_uid>/.. or ../v1/AUTH_<project_uid>/..
:param path: the request path containing a project uid
:return: the project uid
"""
project_uid = taxonomy.UNKNOWN
try:
if common.is_swift_request(
path) and self.strategy.name == 'object-store':
project_uid = self.strategy.get_swift_project_id_from_path(
path)
else:
project_uid = common.get_project_id_from_os_path()
finally:
if project_uid == taxonomy.UNKNOWN:
self.logger.debug(
"unable to obtain target.project_id from request path '{0}'"
.format(path))
else:
self.logger.debug(
"request path '{0}' contains target.project_id '{1}'".
format(path, project_uid))
return project_uid
def get_target_project_id_from_keystone_token_info(self, token_info):
"""
the token info dict contains the service catalog, in which the project specific
endpoint urls per service can be found.
:param token_info: token info dictionary
:return: the project id or unknown
"""
project_id = taxonomy.UNKNOWN
try:
service_catalog = token_info.get('token', {}).get('catalog', [])
if not service_catalog:
raise None
for service in service_catalog:
svc_type = service.get('type', None)
if not svc_type or svc_type != self.service_type:
continue
svc_endpoints = service.get('endpoints', None)
if not svc_endpoints:
continue
project_id = self._get_project_id_from_service_endpoints(
svc_endpoints)
if project_id:
break
except Exception as e:
self.logger.debug(
'unable to get target.project_id from service catalog: ',
str(e))
finally:
if project_id == taxonomy.UNKNOWN:
self.logger.debug(
"unable to get target.project_id '{0}' for service type '{1}' from service catalog"
.format(project_id, self.service_type))
else:
self.logger.debug(
"got target.project_id '{0}' for service type '{1}' from service catalog"
.format(project_id, self.service_type))
return project_id
def _get_project_id_from_service_endpoints(self,
endpoint_list,
endpoint_type=None):
"""
get the project id from an endpoint url for a given type | type = {public,internal,admin}
:param endpoint_list: list of endpoints
:param endpoint_type: optional endpoint type
:return: the project id or unknown
"""
project_id = taxonomy.UNKNOWN
try:
for endpoint in endpoint_list:
url = endpoint.get('url', None)
type = endpoint.get('interface', None)
if not url or not type:
continue
if self.strategy.name == 'object-store':
project_id = self.strategy.get_swift_project_id_from_path(
url)
else:
project_id = common.get_project_id_from_os_path(url)
# break here if endpoint_type is given and types match
if endpoint_type and endpoint_type.lower() == type.lower():
break
# break here if no endpoint_type given but project id was found
elif not endpoint_type and project_id != taxonomy.UNKNOWN:
break
finally:
if project_id == taxonomy.UNKNOWN:
self.logger.debug(
"found no project id in endpoints for service type '{0}'".
format(self.service_type))
else:
self.logger.debug(
"found target project id '{0}' in endpoints for service type '{1}'"
.format(project_id, self.service_type))
return project_id
def get_project_domain_and_user_id_from_keystone_authentication_request(
self, req):
"""
get project, domain, user id from authentication request.
used in combination with client_addr to determine which client authenticates in which scope
:param req: the request
:return: project_id, domain_id, user_id
"""
project_id = domain_id = user_id = taxonomy.UNKNOWN
try:
if not req.json:
return
json_body_dict = common.load_json_dict(req.json)
if not json_body_dict:
return
project_id = common.find_project_id_in_auth_dict(json_body_dict)
domain_id = common.find_domain_id_in_auth_dict(json_body_dict)
user_id = common.find_user_id_in_auth_dict(json_body_dict)
except Exception as e:
self.logger.debug(
'unable to parse keystone authentication request body: {0}'.
format(str(e)))
finally:
return project_id, domain_id, user_id
def get_target_account_container_id_from_request(self, req):
"""
get swift account id, container name from request
:param req: the request
:return: account uid, container name or unknown
"""
# break here if we don't have the object-store strategy
if self.strategy.name != 'object-store':
return taxonomy.UNKNOWN, taxonomy.UNKNOWN
account_id, container_id, _ = self.strategy.get_swift_account_container_object_id_from_path(
req.path)
return account_id, container_id
def determine_target_type_uri(self, req):
"""
determine the target type uri as per concrete strategy
:param req: the request
:return: the target type uri or taxonomy.UNKNOWN
"""
target_type_uri = self.strategy.determine_target_type_uri(req)
self.logger.debug(
"target type URI of requests '{0} {1}' is '{2}'".format(
req.method, req.path, target_type_uri))
return target_type_uri
def determine_cadf_action(self, req, target_type_uri=None):
"""
attempts to determine the cadf action for a request in the following order:
(1) return custom action if one is configured
(2) if /action, /os-instance-action request, return action from request body
(3) return action based on request method
:param custom_action_config: configuration of custom actions
:param target_type_uri: the target type URI
:param req: the request
:return: the cadf action or unknown
"""
cadf_action = self.strategy.determine_cadf_action(req, target_type_uri)
self.logger.debug("cadf action for '{0} {1}' is '{2}'".format(
req.method, req.path, cadf_action))
return cadf_action
class StatsdMiddleware(object):
def __init__(self,
app,
statsd_host='localhost',
statsd_port='8125',
statsd_prefix='openstack',
statsd_replace='id'):
self.app = app
self.replace_strategy = _ReplaceStrategy(
os.getenv('STATSD_REPLACE', statsd_replace))
self.client = DogStatsd(host=os.getenv('STATSD_HOST', statsd_host),
port=int(os.getenv('STATSD_PORT',
statsd_port)),
namespace=os.getenv('STATSD_PREFIX',
statsd_prefix))
@classmethod
def factory(cls, global_config, **local_config):
def _factory(app):
return cls(app, **local_config)
return _factory
def process_response(self,
start,
environ,
response_wrapper,
exception=None):
self.client.increment('responses_total')
status = response_wrapper.get('status')
if status:
status_code = status.split()[0]
else:
status_code = 'none'
method = environ['REQUEST_METHOD']
# cleanse request path
path = urlparse.urlparse(environ['SCRIPT_NAME'] +
environ['PATH_INFO']).path
# strip extensions
path = splitext(path)[0]
# replace parts of the path with constants based on strategy
path = self.replace_strategy.apply(path)
parts = path.rstrip('\/').split('/')
if exception:
parts.append(exception.__class__.__name__)
api = '/'.join(parts)
self.client.timing('latency_by_api',
time.time() - start,
tags=['method:%s' % method,
'api:%s' % api])
self.client.increment('responses_by_api',
tags=[
'method:%s' % method,
'api:%s' % api,
'status:%s' % status_code
])
def __call__(self, environ, start_response):
response_interception = {}
def start_response_wrapper(status, response_headers, exc_info=None):
response_interception.update(status=status,
response_headers=response_headers,
exc_info=exc_info)
return start_response(status, response_headers, exc_info)
start = time.time()
try:
self.client.open_buffer()
self.client.increment('requests_total')
response = self.app(environ, start_response_wrapper)
try:
for event in response:
yield event
finally:
if hasattr(response, 'close'):
response.close()
self.process_response(start, environ, response_interception)
except Exception as exception:
self.process_response(start, environ, response_interception,
exception)
raise
finally:
self.client.close_buffer()
