def __init__(self, args=None, config_path=None):
logging.Handler.__init__(self)
if args is not None and args != '':
config_path = args
if not os.path.isfile(config_path):
raise Exception('Invalid path to config file.')
with open(config_path) as config_file_obj:
self.config = load(config_file_obj.read())
for key in self.DEFAULT_CONFIG:
setattr(
self, key,
self.config.get('main', {}).get(key, None)
or self.DEFAULT_CONFIG[key])
# Initialize Statsd Client
self.statsd = DogStatsd(host=self.host,
port=self.port,
namespace=self.app_key)
self.publish_templates = self.DEFAULT_PUBLISH_TEMPLATES
publish_templates = self.config.get('publish_templates', {})
self.publish_templates.update(publish_templates)
self.counters = self.config.get('counters', {})
self.gauges = self.config.get('gauges', {})
self.timers = self.config.get('timers', [])
self.histograms = self.config.get('histograms', {})
self.sets = self.config.get('sets', {})
self.timers_start_keys = self._get_timers_keys_list('start')
self.timers_end_keys = self._get_timers_keys_list('end')
self.timers_value_keys = self._get_timers_keys_list('value')
def udf(df: pd.DataFrame) -> pd.Series:
from datadog.dogstatsd import DogStatsd
reporter = (DogStatsd(
host=os.environ["STATSD_HOST"],
port=int(os.environ["STATSD_PORT"]),
telemetry_min_flush_interval=0,
) if os.getenv("STATSD_HOST") and os.getenv("STATSD_PORT") else
DogStatsd())
ds = PandasDataset.from_dataset(df)
result = ds.validate(expectations, result_format="COMPLETE")
valid_rows = pd.Series([True] * df.shape[0])
for check in result.results:
if check.exception_info["raised_exception"]:
# ToDo: probably we should mark all rows as invalid
continue
check_kwargs = check.expectation_config.kwargs
check_kwargs.pop("result_format", None)
check_name = "_".join([check.expectation_config.expectation_type] +
[
str(v) for v in check_kwargs.values()
if isinstance(v, (str, int, float))
])
if ("unexpected_count" in check.result
and check.result["unexpected_count"] > 0):
reporter.increment(
"feast_feature_validation_check_failed",
value=check.result["unexpected_count"],
tags=[
f"feature_table:{os.getenv('FEAST_INGESTION_FEATURE_TABLE', 'unknown')}",
f"project:{os.getenv('FEAST_INGESTION_PROJECT_NAME', 'default')}",
f"check:{check_name}",
],
)
valid_rows.iloc[check.result["unexpected_index_list"]] = False
elif "observed_value" in check.result and check.result[
"observed_value"]:
reporter.gauge(
"feast_feature_validation_observed_value",
value=int(check.result["observed_value"] *
100 # storing as decimal with precision 2
) if not check.success else
0, # nullify everything below threshold
tags=[
f"feature_table:{os.getenv('FEAST_INGESTION_FEATURE_TABLE', 'unknown')}",
f"project:{os.getenv('FEAST_INGESTION_PROJECT_NAME', 'default')}",
f"check:{check_name}",
],
)
return valid_rows
def __init__(self,
host='localhost',
port=8125,
prefix=None,
maxudpsize=512,
ipv6=False):
if ipv6:
_log.warning("DogStatsdAdapter() was 'ipv6'. This is ignored")
self._dd_client = DogStatsd(host=host,
port=port,
namespace=prefix,
max_buffer_size=maxudpsize)
def __init__(self,
host: str = 'localhost',
port: int = 8125,
prefix: str = 'faust-app',
rate: float = 1.0,
**kwargs: Any) -> None:
self.client = DogStatsd(host=host,
port=port,
namespace=prefix,
**kwargs)
self.rate = rate
self.sanitize_re = re.compile(r'[^0-9a-zA-Z_]')
self.re_substitution = '_'
def __init__(self,
app,
statsd_host='localhost',
statsd_port='8125',
statsd_prefix='openstack',
statsd_replace='id'):
self.app = app
self.replace_strategy = _ReplaceStrategy(
os.getenv('STATSD_REPLACE', statsd_replace))
self.client = DogStatsd(host=os.getenv('STATSD_HOST', statsd_host),
port=int(os.getenv('STATSD_PORT',
statsd_port)),
namespace=os.getenv('STATSD_PREFIX',
statsd_prefix))
class DogStatsdMonitoring(AbstractMonitoring):
def __init__(self, config):
super().__init__(config)
self.client = DogStatsd()
def send(self, tags, value):
if len(tags) != 3:
raise AssertionError(
"Datadog monitoring implementation needs 3 tags: 'name', 'what' and 'backup_name'"
)
name, what, backup_name = tags
metric = '{name}.{what}'.format(name=name, what=what)
backup_name_tag = 'backup_name:{}'.format(backup_name)
# The backup_name would be a rather high cardinality metrics series if backups are at all frequent.
# This could be a expensive metric so backup_name is droppped from the tags sent by default
if medusa.utils.evaluate_boolean(self.config.send_backup_name_tag):
self.client.gauge(metric, value, tags=[backup_name_tag])
else:
self.client.gauge(metric, value)
def __init__(
self,
host: str = 'localhost',
port: int = 8125,
socket_path: str = None,
prefix: str = None,
default_tags: Dict[str, Any] = None,
):
"""
This is a wrapper that does primarily this:
- setup connection to statsd server
- wrap measuring methods such that they can be used as various things
(context managers, decorators)
-
:param host: Host of the statsd server
:param port: Port of the statsd server
:param prefix: Default prefix to add to all metrics
:param default_tags: Default tags to add to all metrics
"""
# Setup stats connection
self._statsd = DogStatsd(
host=host,
port=port,
socket_path=socket_path,
constant_tags=_dict_as_statsd_tags(default_tags))
# Add measurement methods
self.increment = self._wrap_measurement_method(
self._statsd.increment,
default_value=1,
prefix=self._join_with_prefix(config.measurement.PREFIX_COUNTER,
prefix),
)
self.decrement = self._wrap_measurement_method(
self._statsd.decrement,
default_value=1,
prefix=self._join_with_prefix(config.measurement.PREFIX_COUNTER,
prefix),
)
self.gauge = self._wrap_measurement_method(
self._statsd.gauge,
prefix=self._join_with_prefix(config.measurement.PREFIX_GAUGE,
prefix))
self.histogram = self._wrap_measurement_method(
self._statsd.histogram,
prefix=self._join_with_prefix(config.measurement.PREFIX_HISTOGRAM,
prefix))
self.timing = self._wrap_measurement_method(
self._statsd.timing,
prefix=self._join_with_prefix(config.measurement.PREFIX_TIMING,
prefix))
self.set = self._wrap_measurement_method(
self._statsd.set,
prefix=self._join_with_prefix(config.measurement.PREFIX_SET,
prefix))
# Our own augmented measurement primitives
self.timer = self._wrap_measurement_method(
self._statsd.timing,
prefix=self._join_with_prefix(config.measurement.PREFIX_TIMING,
prefix),
wrapper=TimerMeasuringPrimitive,
)
self.counter = self._wrap_measurement_method(
self._statsd.increment,
prefix=self._join_with_prefix(config.measurement.PREFIX_COUNTER,
prefix),
wrapper=CounterMeasuringPrimitive,
)
class MeasureWrapper:
"""
Wraps connection to the statsd server and creates wrapped measuring methods
on top of the class.
Provided you create an instance Measure = MeasureWrapper(*args), you can
then use it like this:
Measure.increment('metric')(value)
or with context managers:
with Measure.increment('metric') as m:
m(value)
or with function decorators
@Measure.increment('metric')
def my_method(blah, measuring_context):
measuring_context(value)
"""
_statsd = None
# Statsd primitives
decrement = None # type: MeasuringPrimitive
gauge = None # type: MeasuringPrimitive
histogram = None # type: MeasuringPrimitive
increment = None # type: MeasuringPrimitive
set = None # type: MeasuringPrimitive
timing = None # type: MeasuringPrimitive
# Datadogs primitives
# TODO: include them too? Will be easy
# Our own little bit more interesting measuring primitives
counter = None # type: CounterMeasuringPrimitive
timer = None # type: TimerMeasuringPrimitive
def __init__(
self,
host: str = 'localhost',
port: int = 8125,
socket_path: str = None,
prefix: str = None,
default_tags: Dict[str, Any] = None,
):
"""
This is a wrapper that does primarily this:
- setup connection to statsd server
- wrap measuring methods such that they can be used as various things
(context managers, decorators)
-
:param host: Host of the statsd server
:param port: Port of the statsd server
:param prefix: Default prefix to add to all metrics
:param default_tags: Default tags to add to all metrics
"""
# Setup stats connection
self._statsd = DogStatsd(
host=host,
port=port,
socket_path=socket_path,
constant_tags=_dict_as_statsd_tags(default_tags))
# Add measurement methods
self.increment = self._wrap_measurement_method(
self._statsd.increment,
default_value=1,
prefix=self._join_with_prefix(config.measurement.PREFIX_COUNTER,
prefix),
)
self.decrement = self._wrap_measurement_method(
self._statsd.decrement,
default_value=1,
prefix=self._join_with_prefix(config.measurement.PREFIX_COUNTER,
prefix),
)
self.gauge = self._wrap_measurement_method(
self._statsd.gauge,
prefix=self._join_with_prefix(config.measurement.PREFIX_GAUGE,
prefix))
self.histogram = self._wrap_measurement_method(
self._statsd.histogram,
prefix=self._join_with_prefix(config.measurement.PREFIX_HISTOGRAM,
prefix))
self.timing = self._wrap_measurement_method(
self._statsd.timing,
prefix=self._join_with_prefix(config.measurement.PREFIX_TIMING,
prefix))
self.set = self._wrap_measurement_method(
self._statsd.set,
prefix=self._join_with_prefix(config.measurement.PREFIX_SET,
prefix))
# Our own augmented measurement primitives
self.timer = self._wrap_measurement_method(
self._statsd.timing,
prefix=self._join_with_prefix(config.measurement.PREFIX_TIMING,
prefix),
wrapper=TimerMeasuringPrimitive,
)
self.counter = self._wrap_measurement_method(
self._statsd.increment,
prefix=self._join_with_prefix(config.measurement.PREFIX_COUNTER,
prefix),
wrapper=CounterMeasuringPrimitive,
)
def __enter__(self):
self._statsd.__enter__()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._statsd.__exit__(exc_type, exc_val, exc_tb)
def _wrap_measurement_method(self,
func,
prefix,
default_value=None,
wrapper=None):
"""
We need to wrap the singular measurement function with our
MeasuringPrimitive (or a subclass thereof) class, so that we can support
various interfaces on top of it.
If the measurements are disabled, we just replace the statsd function do
nothing.
:param function func: The function to be wrapped
:param string prefix: Common metric prefix
:param any default_value: Default value for the metric
:param MeasuringPrimitive wrapper: The wrapper class to use
:return function: The partial to be called on the MeasuringPrimitive
constructor
"""
return functools.partial(wrapper or MeasuringPrimitive, func, prefix,
default_value)
def _join_with_prefix(self, value_prefix, global_prefix):
"""
Joins prefixes together, useful for combining global and metric type
specific prefix
"""
return '.'.join(filter(None, [global_prefix, value_prefix]))
def __init__(self, app, config, logger=logging.getLogger(__name__)):
self.logger = logger
self.app = app
self.wsgi_config = config
self.watcher_config = {}
self.cadf_service_name = self.wsgi_config.get('cadf_service_name',
None)
self.service_type = self.wsgi_config.get('service_type',
taxonomy.UNKNOWN)
# get the project uid from the request path or from the token (default)
self.is_project_id_from_path = common.string_to_bool(
self.wsgi_config.get('target_project_id_from_path', 'False'))
# get the project id from the service catalog (see documentation on keystone auth_token middleware)
self.is_project_id_from_service_catalog = common.string_to_bool(
self.wsgi_config.get('target_project_id_from_service_catalog',
'False'))
# whether to include the target project id in the metrics
self.is_include_target_project_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_target_project_id_in_metric',
'True'))
# whether to include the target domain id in the metrics
self.is_include_target_domain_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_target_domain_id_in_metric', 'True'))
# whether to include the initiator user id in the metrics
self.is_include_initiator_user_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_initiator_user_id_in_metric',
'False'))
config_file_path = config.get('config_file', None)
if config_file_path:
try:
self.watcher_config = load_config(config_file_path)
except errors.ConfigError as e:
self.logger.debug("custom actions not available: %s", str(e))
custom_action_config = self.watcher_config.get('custom_actions', {})
path_keywords = self.watcher_config.get('path_keywords', {})
keyword_exclusions = self.watcher_config.get('keyword_exclusions', {})
regex_mapping = self.watcher_config.get('regex_path_mapping', {})
# init the strategy used to determine the target type uri
strat = STRATEGIES.get(self.service_type, strategies.BaseCADFStrategy)
# set custom prefix to target type URI or use defaults
target_type_uri_prefix = common.SERVICE_TYPE_CADF_PREFIX_MAP.get(
self.service_type, 'service/{0}'.format(self.service_type))
if self.cadf_service_name:
target_type_uri_prefix = self.cadf_service_name
strategy = strat(target_type_uri_prefix=target_type_uri_prefix,
path_keywords=path_keywords,
keyword_exclusions=keyword_exclusions,
custom_action_config=custom_action_config,
regex_mapping=regex_mapping)
self.strategy = strategy
self.metric_client = DogStatsd(
host=self.wsgi_config.get("statsd_host", "127.0.0.1"),
port=int(self.wsgi_config.get("statsd_port", 9125)),
namespace=self.wsgi_config.get("statsd_namespace",
"openstack_watcher"))
class DogStatsdMetrics(Metrics):
def __init__(self,
id,
prefix=None,
tags=None,
host="127.0.0.1",
port=8125):
self.id = id
self.prefix = prefix
self.tags = tags or {}
self.host = host
self.port = port
self.tags["instance"] = id
def setup(self):
from datadog.dogstatsd import DogStatsd
self.client = DogStatsd(host=self.host, port=self.port)
def gauge(self, metric, value, tags=None, sample_rate=1):
self.client.gauge(
self._get_key(metric),
value,
sample_rate=sample_rate,
tags=self._get_tags(tags),
)
def increment(self, metric, value=1, tags=None, sample_rate=1):
self.client.increment(
self._get_key(metric),
value,
sample_rate=sample_rate,
tags=self._get_tags(tags),
)
def decrement(self, metric, value=1, tags=None, sample_rate=1):
self.client.decrement(
self._get_key(metric),
value,
sample_rate=sample_rate,
tags=self._get_tags(tags),
)
def histogram(self, metric, value, tags=None, sample_rate=1):
self.client.histogram(
self._get_key(metric),
value,
sample_rate=sample_rate,
tags=self._get_tags(tags),
)
def timing(self, metric, value, tags=None, sample_rate=1):
self.client.timing(
self._get_key(metric),
value,
sample_rate=sample_rate,
tags=self._get_tags(tags),
)
def timed(self, metric, tags=None, sample_rate=1, use_ms=None):
self.client.timed(
self._get_key(metric),
sample_rate=sample_rate,
tags=self._get_tags(tags),
use_ms=use_ms,
)
class DatadogStatsClient:
"""Statsd compliant datadog client."""
def __init__(self,
host: str = 'localhost',
port: int = 8125,
prefix: str = 'faust-app',
rate: float = 1.0,
**kwargs: Any) -> None:
self.client = DogStatsd(host=host,
port=port,
namespace=prefix,
**kwargs)
self.rate = rate
self.sanitize_re = re.compile(r'[^0-9a-zA-Z_]')
self.re_substitution = '_'
def gauge(self, metric: str, value: float, labels: Dict = None) -> None:
self.client.gauge(
metric,
value=value,
tags=self._encode_labels(labels),
sample_rate=self.rate,
)
def increment(self,
metric: str,
value: float = 1.0,
labels: Dict = None) -> None:
self.client.increment(
metric,
value=value,
tags=self._encode_labels(labels),
sample_rate=self.rate,
)
def incr(self, metric: str, count: int = 1) -> None:
"""Statsd compatibility."""
self.increment(metric, value=count)
def decrement(self,
metric: str,
value: float = 1.0,
labels: Dict = None) -> float:
return self.client.decrement(
metric,
value=value,
tags=self._encode_labels(labels),
sample_rate=self.rate,
)
def decr(self, metric: str, count: float = 1.0) -> None:
"""Statsd compatibility."""
self.decrement(metric, value=count)
def timing(self, metric: str, value: float, labels: Dict = None) -> None:
self.client.timing(
metric,
value=value,
tags=self._encode_labels(labels),
sample_rate=self.rate,
)
def timed(self,
metric: str = None,
labels: Dict = None,
use_ms: bool = None) -> float:
return self.client.timed(
metric=metric,
tags=self._encode_labels(labels),
sample_rate=self.rate,
use_ms=use_ms,
)
def histogram(self,
metric: str,
value: float,
labels: Dict = None) -> None:
self.client.histogram(
metric,
value=value,
tags=self._encode_labels(labels),
sample_rate=self.rate,
)
def _encode_labels(self, labels: Optional[Dict]) -> Optional[List[str]]:
def sanitize(s: str) -> str:
return self.sanitize_re.sub(self.re_substitution, str(s))
return [f'{sanitize(k)}:{sanitize(v)}'
for k, v in labels.items()] if labels else None
class StatsdMiddleware(object):
def __init__(self,
app,
statsd_host='localhost',
statsd_port='8125',
statsd_prefix='openstack',
statsd_replace='id'):
self.app = app
self.replace_strategy = _ReplaceStrategy(
os.getenv('STATSD_REPLACE', statsd_replace))
self.client = DogStatsd(host=os.getenv('STATSD_HOST', statsd_host),
port=int(os.getenv('STATSD_PORT',
statsd_port)),
namespace=os.getenv('STATSD_PREFIX',
statsd_prefix))
@classmethod
def factory(cls, global_config, **local_config):
def _factory(app):
return cls(app, **local_config)
return _factory
def process_response(self,
start,
environ,
response_wrapper,
exception=None):
self.client.increment('responses_total')
status = response_wrapper.get('status')
if status:
status_code = status.split()[0]
else:
status_code = 'none'
method = environ['REQUEST_METHOD']
# cleanse request path
path = urlparse.urlparse(environ['SCRIPT_NAME'] +
environ['PATH_INFO']).path
# strip extensions
path = splitext(path)[0]
# replace parts of the path with constants based on strategy
path = self.replace_strategy.apply(path)
parts = path.rstrip('\/').split('/')
if exception:
parts.append(exception.__class__.__name__)
api = '/'.join(parts)
self.client.timing('latency_by_api',
time.time() - start,
tags=['method:%s' % method,
'api:%s' % api])
self.client.increment('responses_by_api',
tags=[
'method:%s' % method,
'api:%s' % api,
'status:%s' % status_code
])
def __call__(self, environ, start_response):
response_interception = {}
def start_response_wrapper(status, response_headers, exc_info=None):
response_interception.update(status=status,
response_headers=response_headers,
exc_info=exc_info)
return start_response(status, response_headers, exc_info)
start = time.time()
try:
self.client.open_buffer()
self.client.increment('requests_total')
response = self.app(environ, start_response_wrapper)
try:
for event in response:
yield event
finally:
if hasattr(response, 'close'):
response.close()
self.process_response(start, environ, response_interception)
except Exception as exception:
self.process_response(start, environ, response_interception,
exception)
raise
finally:
self.client.close_buffer()
def _create_statsd(*args, **kwargs):
# testing mock point
return DogStatsd(*args, **kwargs)
def __init__(self, config):
super().__init__(config)
self.client = DogStatsd()
import os
from datadog import initialize
from datadog.dogstatsd import DogStatsd
import config
root = os.path.dirname(os.path.abspath(__file__))
conf = config.Config(os.path.join(root, "config.ini"))
dogstatsd = DogStatsd(host='localhost',
port=conf.datadog_dogstatsd_port,
constant_tags=conf.datadog_tags)
class DogStatsdAdapter(object):
"""
A wrapper around DogStatsd that supports the full statsd.StatsClient interface
Note that `tags` is available on all these methods, but this is not supported by statsd.StatsClient. It has been
added, but should only be used when you are CERTAIN that you will be using this class (or something similar)
"""
def __init__(self,
host='localhost',
port=8125,
prefix=None,
maxudpsize=512,
ipv6=False):
if ipv6:
_log.warning("DogStatsdAdapter() was 'ipv6'. This is ignored")
self._dd_client = DogStatsd(host=host,
port=port,
namespace=prefix,
max_buffer_size=maxudpsize)
def timer(self, stat, rate=1, tags=None):
self._dd_client.timed(metric=stat,
sample_rate=rate,
use_ms=True,
tags=tags)
def timing(self, stat, delta, rate=1, tags=None):
"""Send new timing information. `delta` is in milliseconds."""
self._dd_client.timing(metric=stat,
value=delta,
sample_rate=rate,
tags=tags)
def incr(self, stat, count=1, rate=1, tags=None):
"""Increment a stat by `count`."""
self._dd_client.increment(metric=stat,
value=count,
sample_rate=rate,
tags=tags)
def decr(self, stat, count=1, rate=1, tags=None):
"""Decrement a stat by `count`."""
self._dd_client.decrement(metric=stat,
value=count,
sample_rate=rate,
tags=tags)
def gauge(self, stat, value, rate=1, delta=False, tags=None):
"""Set a gauge value."""
self._dd_client.gauge(metric=stat,
value=value,
sample_rate=rate,
tags=tags)
if delta:
_log.warning(
"DogStatsdAdapter was passed a gauge with 'delta' set. This is ignored in datadog"
)
def set(self, stat, value, rate=1, tags=None):
"""Set a set value."""
self._dd_client.set(metric=stat,
value=value,
sample_rate=rate,
tags=tags)
def histogram(self, stat, value, rate=1, tags=None):
"""
Sample a histogram value, optionally setting tags and a sample rate.
This is not supported by statsd.StatsClient. Use with caution!
"""
self._dd_client.set(metric=stat,
value=value,
sample_rate=rate,
tags=tags)
pass
def __exit__(self, type, value, traceback):
pass
def __call__(self, func):
def wrapped(*args, **kwargs):
return func(*args, **kwargs)
return wrapped
def timed(*args, **kwargs):
return WithDecorator()
stats.timed = timed
else:
from datadog.dogstatsd import DogStatsd
stats = DogStatsd(host=os.getenv('STATSD_HOST', 'localhost'),
port=int(os.getenv('STATSD_PORT', 9125)),
namespace=os.getenv('STATSD_PREFIX', 'openstack')
)
##
# oslo.vmware.vim_util
def _get_token(retrieve_result):
"""Get token from result to obtain next set of results.
:retrieve_result: Result of RetrievePropertiesEx API call
:returns: token to obtain next set of results; None if no more results.
"""
return getattr(retrieve_result, 'token', None)
def cancel_retrieval(si, retrieve_result):
def _get_client(self, host, port, namespace):
return DogStatsd(host=host, port=port, namespace=namespace)
class OpenStackRateLimitMiddleware(object):
"""
OpenStack Rate Limit Middleware enforces configurable rate limits.
Per combination of:
service ( compute, identity, object-store, .. )
scope ( initiator|target project uid, initiator host address )
target_type_uri ( service/compute/servers, service/storage/block/volumes,.. )
action ( create, read, update, delete, authenticate, .. )
"""
def __init__(self, app, **conf):
self.app = app
# Configuration via paste.ini.
self.__conf = conf
self.logger = log.Logger(conf.get('log_name', __name__))
# StatsD is used to emit metrics.
statsd_host = self.__conf.get('statsd_host', '127.0.0.1')
statsd_port = common.to_int(self.__conf.get('statsd_port', 9125))
statsd_prefix = self.__conf.get('statsd_prefix',
common.Constants.metric_prefix)
# Init StatsD client.
self.metricsClient = DogStatsd(
host=os.getenv('STATSD_HOST', statsd_host),
port=int(os.getenv('STATSD_PORT', statsd_port)),
namespace=os.getenv('STATSD_PREFIX', statsd_prefix))
# Get backend configuration.
# Backend is used to store count of requests.
self.backend_host = self.__conf.get('backend_host', '127.0.0.1')
self.backend_port = common.to_int(self.__conf.get('backend_port'),
6379)
self.logger.debug("using backend '{0}' on '{1}:{2}'".format(
'redis', self.backend_host, self.backend_port))
backend_timeout_seconds = common.to_int(
self.__conf.get('backend_timeout_seconds'), 20)
backend_max_connections = common.to_int(
self.__conf.get('backend_max_connections'), 100)
# Load configuration file.
self.config = {}
config_file = self.__conf.get('config_file', None)
if config_file:
try:
self.config = common.load_config(config_file)
except errors.ConfigError as e:
self.logger.warning("error loading configuration: {0}".format(
str(e)))
self.service_type = self.__conf.get('service_type', None)
# This is required to trim the prefix from the target_type_uri.
# Example:
# service_type = identity
# cadf_service_name = data/security
# target_type_uri = data/security/auth/tokens -> auth/tokens
self.cadf_service_name = self.__conf.get('cadf_service_name', None)
if common.is_none_or_unknown(self.cadf_service_name):
self.cadf_service_name = common.CADF_SERVICE_TYPE_PREFIX_MAP.get(
self.service_type, None)
# Use configured parameters or ensure defaults.
max_sleep_time_seconds = common.to_int(
self.__conf.get(common.Constants.max_sleep_time_seconds), 20)
log_sleep_time_seconds = common.to_int(
self.__conf.get(common.Constants.log_sleep_time_seconds), 10)
# Setup ratelimit and blacklist response.
self._setup_response()
# White-/blacklist can contain project, domain, user ids or the client ip address.
# Don't apply rate limits to localhost.
default_whitelist = ['127.0.0.1', 'localhost']
config_whitelist = self.config.get('whitelist', [])
self.whitelist = default_whitelist + config_whitelist
self.whitelist_users = self.config.get('whitelist_users', [])
self.blacklist = self.config.get('blacklist', [])
self.blacklist_users = self.config.get('blacklist_users', [])
# Mapping of potentially multiple CADF actions to one action.
self.rate_limit_groups = self.config.get('groups', {})
# Configurable scope in which a rate limit is applied. Defaults to initiator project id.
# Rate limits are applied based on the tuple of (rate_limit_by, action, target_type_uri).
self.rate_limit_by = self.__conf.get(
'rate_limit_by', common.Constants.initiator_project_id)
# Accuracy of the request timestamps used. Defaults to nanosecond accuracy.
clock_accuracy = int(
1 / units.Units.parse(self.__conf.get('clock_accuracy', '1ns')))
self.backend = rate_limit_backend.RedisBackend(
host=self.backend_host,
port=self.backend_port,
rate_limit_response=self.ratelimit_response,
max_sleep_time_seconds=max_sleep_time_seconds,
log_sleep_time_seconds=log_sleep_time_seconds,
timeout_seconds=backend_timeout_seconds,
max_connections=backend_max_connections,
clock_accuracy=clock_accuracy,
)
# Test if the backend is ready.
is_available, msg = self.backend.is_available()
if not is_available:
self.logger.warning(
"rate limit not possible. the backend is not available: {0}".
format(msg))
# Provider for rate limits. Defaults to configuration file.
# Also supports Limes.
configuration_ratelimit_provider = provider.ConfigurationRateLimitProvider(
service_type=self.service_type)
# Force load of rate limits from configuration file.
configuration_ratelimit_provider.read_rate_limits_from_config(
config_file)
self.ratelimit_provider = configuration_ratelimit_provider
# If limes is enabled and we want to rate limit by initiator|target project id,
# Set LimesRateLimitProvider as the provider for rate limits.
limes_enabled = self.__conf.get('limes_enabled', False)
if limes_enabled:
self.__setup_limes_ratelimit_provider()
self.logger.info("OpenStack Rate Limit Middleware ready for requests.")
def _setup_response(self):
"""Setup configurable RateLimitExceededResponse and BlacklistResponse."""
# Default responses.
ratelimit_response = response.RateLimitExceededResponse()
blacklist_response = response.BlacklistResponse()
# Overwrite default responses if custom ones are configured.
try:
ratelimit_response_config = self.config.get(
common.Constants.ratelimit_response)
if ratelimit_response_config:
status, status_code, headers, body, json_body = \
response.response_parameters_from_config(ratelimit_response_config)
# Only create custom response if all parameters are given.
if status and status_code and (body or json_body):
ratelimit_response = response.RateLimitExceededResponse(
status=status,
status_code=status_code,
headerlist=headers,
body=body,
json_body=json_body)
blacklist_response_config = self.config.get(
common.Constants.blacklist_response)
if blacklist_response_config:
status, status_code, headers, body, json_body = \
response.response_parameters_from_config(blacklist_response_config)
# Only create custom response if all parameters are given.
if status and status_code and (body or json_body):
blacklist_response = response.BlacklistResponse(
status=status,
status_code=status_code,
headerlist=headers,
body=body,
json_body=json_body)
except Exception as e:
self.logger.debug(
"error configuring custom responses. falling back to defaults: {0}"
.format(str(e)))
finally:
self.ratelimit_response = ratelimit_response
self.blacklist_response = blacklist_response
def __setup_limes_ratelimit_provider(self):
"""Setup Limes as provider for rate limits. If not successful fallback to configuration file."""
try:
limes_ratelimit_provider = provider.LimesRateLimitProvider(
service_type=self.service_type,
redis_host=self.backend_host,
redis_port=self.backend_port,
refresh_interval_seconds=self.__conf.get(
common.Constants.limes_refresh_interval_seconds, 300),
limes_api_uri=self.__conf.get(common.Constants.limes_api_uri),
auth_url=self.__conf.get('identity_auth_url'),
username=self.__conf.get('username'),
user_domain_name=self.__conf.get('user_domain_name'),
password=self.__conf.get('password'),
domain_name=self.__conf.get('domain_name'))
self.ratelimit_provider = limes_ratelimit_provider
except Exception as e:
self.logger.debug(
"failed to setup limes rate limit provider: {0}".format(
str(e)))
@classmethod
def factory(cls, global_config, **local_config):
conf = global_config.copy()
conf.update(local_config)
def limiter(app):
return cls(app, **conf)
return limiter
def _rate_limit(self, scope, action, target_type_uri, **kwargs):
"""
Check the whitelist, blacklist, global and local ratelimits.
:param scope: the scope of the request
:param action: the action of the request
:param target_type_uri: the target type URI of the response
:return: None or BlacklistResponse or RateLimitResponse
"""
# Labels used for all metrics.
metric_labels = [
'service:{0}'.format(self.service_type),
'service_name:{0}'.format(self.cadf_service_name),
'action:{0}'.format(action),
'{0}:{1}'.format(self.rate_limit_by, scope),
'target_type_uri:{0}'.format(target_type_uri)
]
global_metric_labels = metric_labels + ['level:global']
local_metric_labels = metric_labels + ['level:local']
# Check whether a set of CADF actions are accounted together.
new_action = self.get_action_from_rate_limit_groups(action)
if not common.is_none_or_unknown(new_action):
action = new_action
metric_labels.append('action_group:{0}'.format(action))
# Get CADF service name and trim from target_type_uri.
trimmed_target_type_uri = target_type_uri
if not common.is_none_or_unknown(self.cadf_service_name):
trimmed_target_type_uri = self._trim_cadf_service_prefix_from_target_type_uri(
self.cadf_service_name, target_type_uri)
username = kwargs.get('username', None)
# If we have the username: Check whether the user is white- or blacklisted.
if username:
metric_labels.append('initiator_user_name:{}'.format(username))
if self.is_user_whitelisted(username):
self.logger.debug(
"user {0} is whitelisted. skipping rate limit".format(
username))
self.metricsClient.increment(
common.Constants.metric_requests_whitelisted_total,
tags=metric_labels)
return None
if self.is_user_blacklisted(username):
self.logger.debug(
"user {0} is blacklisted. returning BlacklistResponse".
format(username))
self.metricsClient.increment(
common.Constants.metric_requests_blacklisted_total,
tags=metric_labels)
return self.blacklist_response
# The key of the scope in the format $domainName/projectName.
scope_name_key = kwargs.get('scope_name_key', None)
if scope_name_key:
metric_labels.append(
'initiator_project_name:{}'.format(scope_name_key))
# Check whitelist. If scope is whitelisted break here and don't apply any rate limits.
if self.is_scope_whitelisted(scope) or self.is_scope_whitelisted(
scope_name_key):
self.logger.debug(
"scope {0} (key: {1}) is whitelisted. skipping rate limit".
format(scope, scope_name_key))
self.metricsClient.increment(
common.Constants.metric_requests_whitelisted_total,
tags=metric_labels)
return None
# Check blacklist. If scope is blacklisted return BlacklistResponse.
if self.is_scope_blacklisted(scope) or self.is_scope_blacklisted(
scope_name_key):
self.logger.debug(
"scope {0} (key: {1}) is blacklisted. returning BlacklistResponse"
.format(scope, scope_name_key))
self.metricsClient.increment(
common.Constants.metric_requests_blacklisted_total,
tags=metric_labels)
return self.blacklist_response
# Get global rate limits from the provider.
global_rate_limit = self.ratelimit_provider.get_global_rate_limits(
action, trimmed_target_type_uri)
# Don't rate limit if limit=-1 or unknown.
if not common.is_unlimited(global_rate_limit):
self.logger.debug(
"global rate limit configured for request with action '{0}', target type URI '{1}': '{2}'"
.format(action, target_type_uri, global_rate_limit))
# Check global rate limits.
# Global rate limits enforce a backend protection by counting all requests independent of their scope.
rate_limit_response = self.backend.rate_limit(
scope=None,
action=action,
target_type_uri=trimmed_target_type_uri,
max_rate_string=global_rate_limit)
if rate_limit_response:
self.metricsClient.increment(
common.Constants.metric_requests_ratelimit_total,
tags=global_metric_labels)
return rate_limit_response
# Get local (for a certain scope) rate limits from provider.
local_rate_limit = self.ratelimit_provider.get_local_rate_limits(
scope, action, trimmed_target_type_uri)
# Don't rate limit for rate_limit=-1 or if unknown.
if not common.is_unlimited(local_rate_limit):
self.logger.debug(
"local rate limit configured for request with action '{0}', target type URI '{1}', scope '{2}': '{3}'"
.format(action, target_type_uri, scope, local_rate_limit))
# Check local (for a specific scope) rate limits.
rate_limit_response = self.backend.rate_limit(
scope=scope,
action=action,
target_type_uri=trimmed_target_type_uri,
max_rate_string=local_rate_limit)
if rate_limit_response:
self.metricsClient.increment(
common.Constants.metric_requests_ratelimit_total,
tags=local_metric_labels)
return rate_limit_response
return None
def __call__(self, environ, start_response):
"""
WSGI entry point. Wraps environ in webob.Request.
:param environ: the WSGI environment dict
:param start_response: WSGI callable
"""
# Save the app's response so it can be returned easily.
resp = self.app
try:
self.metricsClient.open_buffer()
# If the service type and/or service name is not configured,
# attempt to extract watcher classification from environ and set it.
self._set_service_type_and_name(environ)
# Get openstack-watcher-middleware classification from requests environ.
scope, action, target_type_uri = self.get_scope_action_target_type_uri_from_environ(
environ)
# Don't rate limit if any of scope, action, target type URI cannot be determined.
if common.is_none_or_unknown(scope) or \
common.is_none_or_unknown(action) or \
common.is_none_or_unknown(target_type_uri):
path = str(environ.get('PATH_INFO', common.Constants.unknown))
method = str(
environ.get('REQUEST_METHOD', common.Constants.unknown))
self.logger.debug(
"unknown request: action: {0}, target_type_uri: {1}, scope: {2}, method: {3}, path: {4}"
.format(action, target_type_uri, scope, method, path))
self.metricsClient.increment(
common.Constants.metric_requests_unknown_classification,
tags=[
'service:{0}'.format(self.service_type),
'service_name:{0}'.format(self.cadf_service_name),
])
return
# Returns a RateLimitResponse or BlacklistResponse or None, in which case the original response is returned.
rate_limit_response = self._rate_limit(
scope=scope,
action=action,
target_type_uri=target_type_uri,
scope_name_key=self._get_scope_name_key_from_environ(environ),
username=self._get_username_from_environ(environ),
)
if rate_limit_response:
rate_limit_response.set_environ(environ)
resp = rate_limit_response
except Exception as e:
self.metricsClient.increment(common.Constants.metric_errors_total)
self.logger.debug("checking rate limits failed with: {0}".format(
str(e)))
finally:
self.metricsClient.close_buffer()
return resp(environ, start_response)
def is_scope_blacklisted(self, key_to_check):
"""
Check whether a scope (user_id, project_id or client ip) is blacklisted.
:param key_to_check: the user, project uid or client ip
:return: bool whether the key is blacklisted
"""
for entry in self.blacklist:
if entry == key_to_check:
return True
return False
def is_user_blacklisted(self, user_to_check):
"""
Check whether a user is blacklisted.
:param user_to_check: the name of the user to check
:return: bool whether user is blacklisted
"""
for u in self.blacklist_users:
if str(u).lower() == str(user_to_check).lower():
return True
return False
def is_scope_whitelisted(self, key_to_check):
"""
Check whether a scope (user_id, project_id or client ip) is whitelisted.
:param key_to_check: the user, project uid or client ip
:return: bool whether the key is whitelisted
"""
for entry in self.whitelist:
if entry == key_to_check:
return True
return False
def is_user_whitelisted(self, user_to_check):
"""
Check whether a user is whitelisted.
:param user_to_check: the name of the user to check
:return: bool whether user is whitelisted
"""
for u in self.whitelist_users:
if str(u).lower() == str(user_to_check).lower():
return True
return False
def get_scope_action_target_type_uri_from_environ(self, environ):
"""
Get the scope, action, target type URI from the request environ.
:param environ: the request environ
:return: tuple of scope, action, target type URI
"""
action = target_type_uri = scope = None
try:
# Get the CADF action.
env_action = environ.get('WATCHER.ACTION')
if not common.is_none_or_unknown(env_action):
action = env_action
# Get the target type URI.
env_target_type_uri = environ.get('WATCHER.TARGET_TYPE_URI')
if not common.is_none_or_unknown(env_target_type_uri):
target_type_uri = env_target_type_uri
# Get scope from request environment, which might be an initiator.project_id, target.project_id, etc. .
env_scope = self._get_scope_from_environ(environ)
if not common.is_none_or_unknown(env_scope):
scope = env_scope
except Exception as e:
self.logger.debug(
"error while getting scope, action, target type URI from environ: {0}"
.format(str(e)))
finally:
self.logger.debug(
'got WATCHER.* attributes from environ: action: {0}, target_type_uri: {1}, scope: {2}'
.format(action, target_type_uri, scope))
return scope, action, target_type_uri
def _get_scope_from_environ(self, environ):
"""
Get the scope from the requests environ.
The scope is configurable and may be the target|initiator project uid or the initiator host address.
Default to initiator project ID.
:param environ: the requests environ
:return: the scope
"""
scope = None
if self.rate_limit_by == common.Constants.target_project_id:
env_scope = environ.get('WATCHER.TARGET_PROJECT_ID', None)
elif self.rate_limit_by == common.Constants.initiator_host_address:
env_scope = environ.get('WATCHER.INITIATOR_HOST_ADDRESS', None)
else:
env_scope = environ.get('WATCHER.INITIATOR_PROJECT_ID', None)
# Ensure the scope is not 'unknown'.
if not common.is_none_or_unknown(env_scope):
scope = env_scope
return scope
def _get_scope_name_key_from_environ(self, environ):
"""
Attempt to build the key '$domainName/$projectName' from the WATCHER attributes found in the request environ.
:param environ: the request environ
:return: the key or None
"""
_domain_name = environ.get('WATCHER.INITIATOR_DOMAIN_NAME', None)
_project_domain_name = environ.get(
'WATCHER.INITIATOR_PROJECT_DOMAIN_NAME', None)
project_name = environ.get('WATCHER.INITIATOR_PROJECT_NAME', None)
domain_name = _project_domain_name or _domain_name
if common.is_none_or_unknown(
project_name) or common.is_none_or_unknown(domain_name):
return None
return '{0}/{1}'.format(domain_name, project_name)
def _get_username_from_environ(self, environ):
"""
Attempt to get username from WATCHER attributes found in request environ.
:param environ: the request environ
:return: the username or None
"""
username = environ.get('WATCHER.INITIATOR_USER_NAME', None)
if common.is_none_or_unknown(username):
return None
return username
def _set_service_type_and_name(self, environ):
"""
Set the service type and name according to the watchers classification passed in the request WSGI environ.
Used if nothing was configured.
:param environ: the request WSGI environment
"""
# Get service type from request environ.
if common.is_none_or_unknown(self.service_type):
svc_type = environ.get('WATCHER.SERVICE_TYPE')
if not common.is_none_or_unknown(svc_type):
self.service_type = svc_type
self.ratelimit_provider.service_type = self.service_type
# set service name from environ
if common.is_none_or_unknown(self.cadf_service_name):
svc_name = environ.get('WATCHER.CADF_SERVICE_NAME')
if not common.is_none_or_unknown(svc_name):
self.cadf_service_name = svc_name
self.ratelimit_provider.cadf_service_name = self.cadf_service_name
def _trim_cadf_service_prefix_from_target_type_uri(self, prefix,
target_type_uri):
"""
Get cadf service name and trim from target_type_uri.
Example:
target_type_uri: service/storage/object/account/container/object
cadf_service_name: service/storage/object
=> trimmed_target_type_uri: account/container/object
:param prefix: the cadf service name prefixing the target_type_uri
:param target_type_uri: the target_type_uri with the prefix
:return: target_type_uri without prefix
"""
target_type_uri_without_prefix = target_type_uri
try:
without_prefix = target_type_uri.split(prefix)
if len(without_prefix) != 2:
raise IndexError
target_type_uri_without_prefix = without_prefix[-1].lstrip('/')
except IndexError as e:
self.logger.warning(
"rate limiting might not be possible. cannot trim prefix '{0}' from target_type_uri '{1}': {2}"
.format(prefix, target_type_uri, str(e)))
finally:
return target_type_uri_without_prefix
def get_action_from_rate_limit_groups(self, action):
"""
Multiple CADF actions can be grouped and accounted as one entity.
:param action: the original CADF action
:return: the original action or action as per grouping
"""
for group in self.rate_limit_groups:
if action in self.rate_limit_groups[group]:
return group
return action
def __init__(self, app, **conf):
self.app = app
# Configuration via paste.ini.
self.__conf = conf
self.logger = log.Logger(conf.get('log_name', __name__))
# StatsD is used to emit metrics.
statsd_host = self.__conf.get('statsd_host', '127.0.0.1')
statsd_port = common.to_int(self.__conf.get('statsd_port', 9125))
statsd_prefix = self.__conf.get('statsd_prefix',
common.Constants.metric_prefix)
# Init StatsD client.
self.metricsClient = DogStatsd(
host=os.getenv('STATSD_HOST', statsd_host),
port=int(os.getenv('STATSD_PORT', statsd_port)),
namespace=os.getenv('STATSD_PREFIX', statsd_prefix))
# Get backend configuration.
# Backend is used to store count of requests.
self.backend_host = self.__conf.get('backend_host', '127.0.0.1')
self.backend_port = common.to_int(self.__conf.get('backend_port'),
6379)
self.logger.debug("using backend '{0}' on '{1}:{2}'".format(
'redis', self.backend_host, self.backend_port))
backend_timeout_seconds = common.to_int(
self.__conf.get('backend_timeout_seconds'), 20)
backend_max_connections = common.to_int(
self.__conf.get('backend_max_connections'), 100)
# Load configuration file.
self.config = {}
config_file = self.__conf.get('config_file', None)
if config_file:
try:
self.config = common.load_config(config_file)
except errors.ConfigError as e:
self.logger.warning("error loading configuration: {0}".format(
str(e)))
self.service_type = self.__conf.get('service_type', None)
# This is required to trim the prefix from the target_type_uri.
# Example:
# service_type = identity
# cadf_service_name = data/security
# target_type_uri = data/security/auth/tokens -> auth/tokens
self.cadf_service_name = self.__conf.get('cadf_service_name', None)
if common.is_none_or_unknown(self.cadf_service_name):
self.cadf_service_name = common.CADF_SERVICE_TYPE_PREFIX_MAP.get(
self.service_type, None)
# Use configured parameters or ensure defaults.
max_sleep_time_seconds = common.to_int(
self.__conf.get(common.Constants.max_sleep_time_seconds), 20)
log_sleep_time_seconds = common.to_int(
self.__conf.get(common.Constants.log_sleep_time_seconds), 10)
# Setup ratelimit and blacklist response.
self._setup_response()
# White-/blacklist can contain project, domain, user ids or the client ip address.
# Don't apply rate limits to localhost.
default_whitelist = ['127.0.0.1', 'localhost']
config_whitelist = self.config.get('whitelist', [])
self.whitelist = default_whitelist + config_whitelist
self.whitelist_users = self.config.get('whitelist_users', [])
self.blacklist = self.config.get('blacklist', [])
self.blacklist_users = self.config.get('blacklist_users', [])
# Mapping of potentially multiple CADF actions to one action.
self.rate_limit_groups = self.config.get('groups', {})
# Configurable scope in which a rate limit is applied. Defaults to initiator project id.
# Rate limits are applied based on the tuple of (rate_limit_by, action, target_type_uri).
self.rate_limit_by = self.__conf.get(
'rate_limit_by', common.Constants.initiator_project_id)
# Accuracy of the request timestamps used. Defaults to nanosecond accuracy.
clock_accuracy = int(
1 / units.Units.parse(self.__conf.get('clock_accuracy', '1ns')))
self.backend = rate_limit_backend.RedisBackend(
host=self.backend_host,
port=self.backend_port,
rate_limit_response=self.ratelimit_response,
max_sleep_time_seconds=max_sleep_time_seconds,
log_sleep_time_seconds=log_sleep_time_seconds,
timeout_seconds=backend_timeout_seconds,
max_connections=backend_max_connections,
clock_accuracy=clock_accuracy,
)
# Test if the backend is ready.
is_available, msg = self.backend.is_available()
if not is_available:
self.logger.warning(
"rate limit not possible. the backend is not available: {0}".
format(msg))
# Provider for rate limits. Defaults to configuration file.
# Also supports Limes.
configuration_ratelimit_provider = provider.ConfigurationRateLimitProvider(
service_type=self.service_type)
# Force load of rate limits from configuration file.
configuration_ratelimit_provider.read_rate_limits_from_config(
config_file)
self.ratelimit_provider = configuration_ratelimit_provider
# If limes is enabled and we want to rate limit by initiator|target project id,
# Set LimesRateLimitProvider as the provider for rate limits.
limes_enabled = self.__conf.get('limes_enabled', False)
if limes_enabled:
self.__setup_limes_ratelimit_provider()
self.logger.info("OpenStack Rate Limit Middleware ready for requests.")
class OpenStackWatcherMiddleware(object):
"""
OpenStack Watcher Middleware
Watches OpenStack traffic and classifies according to CADF standard
"""
def __init__(self, app, config, logger=logging.getLogger(__name__)):
self.logger = logger
self.app = app
self.wsgi_config = config
self.watcher_config = {}
self.cadf_service_name = self.wsgi_config.get('cadf_service_name',
None)
self.service_type = self.wsgi_config.get('service_type',
taxonomy.UNKNOWN)
# get the project uid from the request path or from the token (default)
self.is_project_id_from_path = common.string_to_bool(
self.wsgi_config.get('target_project_id_from_path', 'False'))
# get the project id from the service catalog (see documentation on keystone auth_token middleware)
self.is_project_id_from_service_catalog = common.string_to_bool(
self.wsgi_config.get('target_project_id_from_service_catalog',
'False'))
# whether to include the target project id in the metrics
self.is_include_target_project_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_target_project_id_in_metric',
'True'))
# whether to include the target domain id in the metrics
self.is_include_target_domain_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_target_domain_id_in_metric', 'True'))
# whether to include the initiator user id in the metrics
self.is_include_initiator_user_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_initiator_user_id_in_metric',
'False'))
config_file_path = config.get('config_file', None)
if config_file_path:
try:
self.watcher_config = load_config(config_file_path)
except errors.ConfigError as e:
self.logger.debug("custom actions not available: %s", str(e))
custom_action_config = self.watcher_config.get('custom_actions', {})
path_keywords = self.watcher_config.get('path_keywords', {})
keyword_exclusions = self.watcher_config.get('keyword_exclusions', {})
regex_mapping = self.watcher_config.get('regex_path_mapping', {})
# init the strategy used to determine the target type uri
strat = STRATEGIES.get(self.service_type, strategies.BaseCADFStrategy)
# set custom prefix to target type URI or use defaults
target_type_uri_prefix = common.SERVICE_TYPE_CADF_PREFIX_MAP.get(
self.service_type, 'service/{0}'.format(self.service_type))
if self.cadf_service_name:
target_type_uri_prefix = self.cadf_service_name
strategy = strat(target_type_uri_prefix=target_type_uri_prefix,
path_keywords=path_keywords,
keyword_exclusions=keyword_exclusions,
custom_action_config=custom_action_config,
regex_mapping=regex_mapping)
self.strategy = strategy
self.metric_client = DogStatsd(
host=self.wsgi_config.get("statsd_host", "127.0.0.1"),
port=int(self.wsgi_config.get("statsd_port", 9125)),
namespace=self.wsgi_config.get("statsd_namespace",
"openstack_watcher"))
@classmethod
def factory(cls, global_config, **local_config):
conf = global_config.copy()
conf.update(local_config)
def watcher(app):
return cls(app, conf)
return watcher
def __call__(self, environ, start_response):
"""
WSGI entry point. Wraps environ in webob.Request
:param environ: the WSGI environment dict
:param start_response: WSGI callable
"""
# capture start timestamp
start = time.time()
req = Request(environ)
# determine initiator based on token context
initiator_project_id = self.get_safe_from_environ(
environ, 'HTTP_X_PROJECT_ID')
initiator_project_name = self.get_safe_from_environ(
environ, 'HTTP_X_PROJECT_NAME')
initiator_project_domain_id = self.get_safe_from_environ(
environ, 'HTTP_X_PROJECT_DOMAIN_ID')
initiator_project_domain_name = self.get_safe_from_environ(
environ, 'HTTP_X_PROJECT_DOMAIN_NAME')
initiator_domain_id = self.get_safe_from_environ(
environ, 'HTTP_X_DOMAIN_ID')
initiator_domain_name = self.get_safe_from_environ(
environ, 'HTTP_X_DOMAIN_NAME')
initiator_user_id = self.get_safe_from_environ(environ,
'HTTP_X_USER_ID')
initiator_user_name = self.get_safe_from_environ(
environ, 'HTTP_X_USER_NAME')
initiator_user_domain_id = self.get_safe_from_environ(
environ, 'HTTP_X_USER_DOMAIN_ID')
initiator_user_domain_name = self.get_safe_from_environ(
environ, 'HTTP_X_USER_DOMAIN_NAME')
initiator_host_address = req.client_addr or taxonomy.UNKNOWN
# determine target based on request path or keystone.token_info
target_project_id = taxonomy.UNKNOWN
if self.is_project_id_from_path:
target_project_id = self.get_target_project_uid_from_path(req.path)
elif self.is_project_id_from_service_catalog:
target_project_id = self.get_target_project_id_from_keystone_token_info(
environ.get('keystone.token_info'))
# default target_project_id to initiator_project_id if still unknown
if not target_project_id or target_project_id == taxonomy.UNKNOWN:
target_project_id = initiator_project_id
# determine target.type_uri for request
target_type_uri = self.determine_target_type_uri(req)
# determine cadf_action for request. consider custom action config.
cadf_action = self.determine_cadf_action(req, target_type_uri)
# if authentication request consider project, domain and user in body
if self.service_type == 'identity' and cadf_action == taxonomy.ACTION_AUTHENTICATE:
initiator_project_id, initiator_domain_id, initiator_user_id = \
self.get_project_domain_and_user_id_from_keystone_authentication_request(req)
# set environ for initiator
environ['WATCHER.INITIATOR_PROJECT_ID'] = initiator_project_id
environ['WATCHER.INITIATOR_PROJECT_NAME'] = initiator_project_name
environ[
'WATCHER.INITIATOR_PROJECT_DOMAIN_ID'] = initiator_project_domain_id
environ[
'WATCHER.INITIATOR_PROJECT_DOMAIN_NAME'] = initiator_project_domain_name
environ['WATCHER.INITIATOR_DOMAIN_ID'] = initiator_domain_id
environ['WATCHER.INITIATOR_DOMAIN_NAME'] = initiator_domain_name
environ['WATCHER.INITIATOR_USER_ID'] = initiator_user_id
environ['WATCHER.INITIATOR_USER_NAME'] = initiator_user_name
environ['WATCHER.INITIATOR_USER_DOMAIN_ID'] = initiator_user_domain_id
environ[
'WATCHER.INITIATOR_USER_DOMAIN_NAME'] = initiator_user_domain_name
environ['WATCHER.INITIATOR_HOST_ADDRESS'] = initiator_host_address
# set environ for target
environ['WATCHER.TARGET_PROJECT_ID'] = target_project_id
environ['WATCHER.TARGET_TYPE_URI'] = target_type_uri
# general cadf attributes
environ['WATCHER.ACTION'] = cadf_action
environ['WATCHER.SERVICE_TYPE'] = self.service_type
environ[
'WATCHER.CADF_SERVICE_NAME'] = self.strategy.get_cadf_service_name(
)
# labels applied to all metrics emitted by this middleware
labels = [
"service_name:{0}".format(self.strategy.get_cadf_service_name()),
"service:{0}".format(self.service_type),
"action:{0}".format(cadf_action),
"target_type_uri:{0}".format(target_type_uri),
]
# additional labels not needed in all metrics
detail_labels = [
"initiator_project_id:{0}".format(initiator_project_id),
"initiator_domain_id:{0}".format(initiator_domain_id),
]
detail_labels = labels + detail_labels
# include the target project id in metric
if self.is_include_target_project_id_in_metric:
detail_labels.append(
"target_project_id:{0}".format(target_project_id))
# include initiator user id
if self.is_include_initiator_user_id_in_metric:
detail_labels.append(
"initiator_user_id:{0}".format(initiator_user_id))
# if swift request: determine target.container_id based on request path
if common.is_swift_request(
req.path) or self.service_type == 'object-store':
_, target_container_id = self.get_target_account_container_id_from_request(
req)
environ['WATCHER.TARGET_CONTAINER_ID'] = target_container_id
self.logger.debug(
'got request with initiator_project_id: {0}, initiator_domain_id: {1}, initiator_user_id: {2}, '
'target_project_id: {3}, action: {4}, target_type_uri: {5}'.format(
initiator_project_id, initiator_domain_id, initiator_user_id,
target_project_id, cadf_action, target_type_uri))
# capture the response status
response_wrapper = {}
try:
def _start_response_wrapper(status, headers, exc_info=None):
response_wrapper.update(status=status,
headers=headers,
exc_info=exc_info)
return start_response(status, headers, exc_info)
return self.app(environ, _start_response_wrapper)
finally:
try:
self.metric_client.open_buffer()
status = response_wrapper.get('status')
if status:
status_code = status.split()[0]
else:
status_code = taxonomy.UNKNOWN
labels.append("status:{0}".format(status_code))
detail_labels.append("status:{0}".format(status_code))
self.metric_client.timing(
'api_requests_duration_seconds',
int(round(1000 * (time.time() - start))),
tags=labels)
self.metric_client.increment('api_requests_total',
tags=detail_labels)
except Exception as e:
self.logger.debug("failed to submit metrics for %s: %s" %
(str(labels), str(e)))
finally:
self.metric_client.close_buffer()
def get_safe_from_environ(self, environ, key, default=taxonomy.UNKNOWN):
"""
get value for a key from the environ dict ensuring it's never None or an empty string
:param environ: the request environ
:param key: the key in the environ dictionary
:param default: return value if key not found
:return: the value to the key or default
"""
val = default
try:
v = environ.get(key, default)
if v and v != "":
val = v
except Exception as e:
self.logger.debug("error getting '{0}' from environ: {1}".format(
key, e))
finally:
return val
def get_target_project_uid_from_path(self, path):
"""
get the project uid from the path, which should look like
../v1.2/<project_uid>/.. or ../v1/AUTH_<project_uid>/..
:param path: the request path containing a project uid
:return: the project uid
"""
project_uid = taxonomy.UNKNOWN
try:
if common.is_swift_request(
path) and self.strategy.name == 'object-store':
project_uid = self.strategy.get_swift_project_id_from_path(
path)
else:
project_uid = common.get_project_id_from_os_path()
finally:
if project_uid == taxonomy.UNKNOWN:
self.logger.debug(
"unable to obtain target.project_id from request path '{0}'"
.format(path))
else:
self.logger.debug(
"request path '{0}' contains target.project_id '{1}'".
format(path, project_uid))
return project_uid
def get_target_project_id_from_keystone_token_info(self, token_info):
"""
the token info dict contains the service catalog, in which the project specific
endpoint urls per service can be found.
:param token_info: token info dictionary
:return: the project id or unknown
"""
project_id = taxonomy.UNKNOWN
try:
service_catalog = token_info.get('token', {}).get('catalog', [])
if not service_catalog:
raise None
for service in service_catalog:
svc_type = service.get('type', None)
if not svc_type or svc_type != self.service_type:
continue
svc_endpoints = service.get('endpoints', None)
if not svc_endpoints:
continue
project_id = self._get_project_id_from_service_endpoints(
svc_endpoints)
if project_id:
break
except Exception as e:
self.logger.debug(
'unable to get target.project_id from service catalog: ',
str(e))
finally:
if project_id == taxonomy.UNKNOWN:
self.logger.debug(
"unable to get target.project_id '{0}' for service type '{1}' from service catalog"
.format(project_id, self.service_type))
else:
self.logger.debug(
"got target.project_id '{0}' for service type '{1}' from service catalog"
.format(project_id, self.service_type))
return project_id
def _get_project_id_from_service_endpoints(self,
endpoint_list,
endpoint_type=None):
"""
get the project id from an endpoint url for a given type | type = {public,internal,admin}
:param endpoint_list: list of endpoints
:param endpoint_type: optional endpoint type
:return: the project id or unknown
"""
project_id = taxonomy.UNKNOWN
try:
for endpoint in endpoint_list:
url = endpoint.get('url', None)
type = endpoint.get('interface', None)
if not url or not type:
continue
if self.strategy.name == 'object-store':
project_id = self.strategy.get_swift_project_id_from_path(
url)
else:
project_id = common.get_project_id_from_os_path(url)
# break here if endpoint_type is given and types match
if endpoint_type and endpoint_type.lower() == type.lower():
break
# break here if no endpoint_type given but project id was found
elif not endpoint_type and project_id != taxonomy.UNKNOWN:
break
finally:
if project_id == taxonomy.UNKNOWN:
self.logger.debug(
"found no project id in endpoints for service type '{0}'".
format(self.service_type))
else:
self.logger.debug(
"found target project id '{0}' in endpoints for service type '{1}'"
.format(project_id, self.service_type))
return project_id
def get_project_domain_and_user_id_from_keystone_authentication_request(
self, req):
"""
get project, domain, user id from authentication request.
used in combination with client_addr to determine which client authenticates in which scope
:param req: the request
:return: project_id, domain_id, user_id
"""
project_id = domain_id = user_id = taxonomy.UNKNOWN
try:
if not req.json:
return
json_body_dict = common.load_json_dict(req.json)
if not json_body_dict:
return
project_id = common.find_project_id_in_auth_dict(json_body_dict)
domain_id = common.find_domain_id_in_auth_dict(json_body_dict)
user_id = common.find_user_id_in_auth_dict(json_body_dict)
except Exception as e:
self.logger.debug(
'unable to parse keystone authentication request body: {0}'.
format(str(e)))
finally:
return project_id, domain_id, user_id
def get_target_account_container_id_from_request(self, req):
"""
get swift account id, container name from request
:param req: the request
:return: account uid, container name or unknown
"""
# break here if we don't have the object-store strategy
if self.strategy.name != 'object-store':
return taxonomy.UNKNOWN, taxonomy.UNKNOWN
account_id, container_id, _ = self.strategy.get_swift_account_container_object_id_from_path(
req.path)
return account_id, container_id
def determine_target_type_uri(self, req):
"""
determine the target type uri as per concrete strategy
:param req: the request
:return: the target type uri or taxonomy.UNKNOWN
"""
target_type_uri = self.strategy.determine_target_type_uri(req)
self.logger.debug(
"target type URI of requests '{0} {1}' is '{2}'".format(
req.method, req.path, target_type_uri))
return target_type_uri
def determine_cadf_action(self, req, target_type_uri=None):
"""
attempts to determine the cadf action for a request in the following order:
(1) return custom action if one is configured
(2) if /action, /os-instance-action request, return action from request body
(3) return action based on request method
:param custom_action_config: configuration of custom actions
:param target_type_uri: the target type URI
:param req: the request
:return: the cadf action or unknown
"""
cadf_action = self.strategy.determine_cadf_action(req, target_type_uri)
self.logger.debug("cadf action for '{0} {1}' is '{2}'".format(
req.method, req.path, cadf_action))
return cadf_action
class StatsdHandler(logging.Handler):
DEFAULT_PUBLISH_TEMPLATES = {
'default': ['%(logger)s;%(attr)s;%(metric_name)s']
}
DEFAULT_CONFIG = {
'app_key': 'default_app_key',
'host': 'localhost',
'port': 8125,
'sample_rate': 1,
'disabled': False
}
def __init__(self, args=None, config_path=None):
logging.Handler.__init__(self)
if args is not None and args != '':
config_path = args
if not os.path.isfile(config_path):
raise Exception('Invalid path to config file.')
with open(config_path) as config_file_obj:
self.config = load(config_file_obj.read())
for key in self.DEFAULT_CONFIG:
setattr(
self, key,
self.config.get('main', {}).get(key, None)
or self.DEFAULT_CONFIG[key])
# Initialize Statsd Client
self.statsd = DogStatsd(host=self.host,
port=self.port,
namespace=self.app_key)
self.publish_templates = self.DEFAULT_PUBLISH_TEMPLATES
publish_templates = self.config.get('publish_templates', {})
self.publish_templates.update(publish_templates)
self.counters = self.config.get('counters', {})
self.gauges = self.config.get('gauges', {})
self.timers = self.config.get('timers', [])
self.histograms = self.config.get('histograms', {})
self.sets = self.config.get('sets', {})
self.timers_start_keys = self._get_timers_keys_list('start')
self.timers_end_keys = self._get_timers_keys_list('end')
self.timers_value_keys = self._get_timers_keys_list('value')
def _get_timers_keys_list(self, key_prefix):
keys = []
for t in self.timers:
key = t.get('{}_attr_name'.format(key_prefix), None)
if key is not None:
keys.append(key)
return keys
def _get_timer_params(self,
key_prefix,
value_attr_name=None,
start_attr_name=None,
end_attr_name=None):
for t in self.timers:
if key_prefix == 'start':
if (t.get('start_attr_name', None) == start_attr_name
and t.get('end_attr_name', None) == end_attr_name):
return [
t.get('name', start_attr_name),
t.get('publish_template', 'default')
]
elif key_prefix == 'value':
if (t.get('value_attr_name', None) == value_attr_name):
return [
t.get('name', value_attr_name),
t.get('publish_template', 'default')
]
else:
return None, None
def _publish_count(self, subname, value):
try:
if float(value) > 0:
self.statsd.increment(subname, value)
else:
self.statsd.decrement(subname, value)
except:
pass
def _publish_timer(self, subname, value):
try:
self.statsd.timing(subname, value)
except:
pass
def _process_counter_metrics(self, attr, record):
lookup_value = self.counters[attr].get('lookup_value', 'None')
value = getattr(record, attr, None)
value_type = self.counters[attr].get('value_type', 'key')
value_equals = self.counters[attr].get('value_equals', [])
publish_template = self.counters[attr].get('publish_template',
'default')
if publish_template not in self.publish_templates:
publish_template = 'default'
if value_type == 'value':
counter_subname = attr
counter_value = value if value is not None else 1
elif value_type == 'key':
counter_subname = value if value is not None else lookup_value
if len(value_equals) > 0 and value not in value_equals:
return
counter_value = 1
else:
return
for pt in self.publish_templates[publish_template]:
subname = pt % dict(
logger=record.name, attr=attr, metric_name=counter_subname)
self._publish_count(subname, counter_value)
def _process_timer_metrics(self, attr, record, key_prefix):
if key_prefix == 'start':
start_attr_value = getattr(record, attr, None)
if start_attr_value is None:
return
start_attr_name = attr
end_attr_value = None
for end_attr_name in self.timers_end_keys:
end_attr_value = getattr(record, end_attr_name, None)
if end_attr_value is not None:
try:
timer_value = float(end_attr_value) -\
float(start_attr_value)
timer_name, publish_template =\
self._get_timer_params(
key_prefix, start_attr_name=start_attr_name,
end_attr_name=end_attr_name)
if publish_template not in self.publish_templates:
publish_template = 'default'
if timer_name is not None:
for pt in self.publish_templates[publish_template]:
subname = pt % dict(logger=record.name,
attr='',
metric_name=timer_name)
self._publish_timer(subname, timer_value)
except:
pass
elif key_prefix == 'value':
timer_value = getattr(record, attr, None)
if timer_value is None:
return
timer_name, publish_template = self._get_timer_params(
key_prefix, value_attr_name=attr)
if publish_template not in self.publish_templates:
publish_template = 'default'
if timer_name is not None:
for pt in self.publish_templates[publish_template]:
subname = pt % dict(
logger=record.name, attr='', metric_name=timer_name)
self._publish_timer(subname, timer_value)
def _get_publish_template(self, metric_type, attr):
metric_types_dict = getattr(self, metric_type, {})
if metric_type == 'sets':
publisher = self.statsd.set
elif metric_type == 'gauges':
publisher = self.statsd.gauge
elif metric_type == 'histograms':
publisher = self.statsd.histogram
else:
publisher = None
publish_template =\
metric_types_dict[attr].get('publish_template', 'default')
if publish_template not in self.publish_templates:
publish_template = 'default'
return publish_template, publisher
def _process_metrics(self, metric_type, attr, record):
value = getattr(record, attr, None)
if value is not None:
publish_template, publisher =\
self._get_publish_template(metric_type, attr)
for pt in self.publish_templates[publish_template]:
subname = pt % dict(
logger=record.name, attr=attr, metric_name=attr)
publisher(subname, value)
def emit(self, record):
for attr in dir(record):
if attr in self.counters:
self._process_counter_metrics(attr, record)
elif attr in self.gauges:
self._process_metrics('gauges', attr, record)
elif attr in self.timers_start_keys:
self._process_timer_metrics(attr, record, 'start')
elif attr in self.timers_value_keys:
self._process_timer_metrics(attr, record, 'value')
elif attr in self.histograms:
self._process_metrics('histograms', attr, record)
elif attr in self.sets:
self._process_metrics('sets', attr, record)
else:
continue
def setup(self):
from datadog.dogstatsd import DogStatsd
self.client = DogStatsd(host=self.host, port=self.port)