def udf(df: pd.DataFrame) -> pd.Series:
from datadog.dogstatsd import DogStatsd
reporter = (DogStatsd(
host=os.environ["STATSD_HOST"],
port=int(os.environ["STATSD_PORT"]),
telemetry_min_flush_interval=0,
) if os.getenv("STATSD_HOST") and os.getenv("STATSD_PORT") else
DogStatsd())
ds = PandasDataset.from_dataset(df)
result = ds.validate(expectations, result_format="COMPLETE")
valid_rows = pd.Series([True] * df.shape[0])
for check in result.results:
if check.exception_info["raised_exception"]:
# ToDo: probably we should mark all rows as invalid
continue
check_kwargs = check.expectation_config.kwargs
check_kwargs.pop("result_format", None)
check_name = "_".join([check.expectation_config.expectation_type] +
[
str(v) for v in check_kwargs.values()
if isinstance(v, (str, int, float))
])
if ("unexpected_count" in check.result
and check.result["unexpected_count"] > 0):
reporter.increment(
"feast_feature_validation_check_failed",
value=check.result["unexpected_count"],
tags=[
f"feature_table:{os.getenv('FEAST_INGESTION_FEATURE_TABLE', 'unknown')}",
f"project:{os.getenv('FEAST_INGESTION_PROJECT_NAME', 'default')}",
f"check:{check_name}",
],
)
valid_rows.iloc[check.result["unexpected_index_list"]] = False
elif "observed_value" in check.result and check.result[
"observed_value"]:
reporter.gauge(
"feast_feature_validation_observed_value",
value=int(check.result["observed_value"] *
100 # storing as decimal with precision 2
) if not check.success else
0, # nullify everything below threshold
tags=[
f"feature_table:{os.getenv('FEAST_INGESTION_FEATURE_TABLE', 'unknown')}",
f"project:{os.getenv('FEAST_INGESTION_PROJECT_NAME', 'default')}",
f"check:{check_name}",
],
)
return valid_rows
def __init__(self, args=None, config_path=None):
logging.Handler.__init__(self)
if args is not None and args != '':
config_path = args
if not os.path.isfile(config_path):
raise Exception('Invalid path to config file.')
with open(config_path) as config_file_obj:
self.config = load(config_file_obj.read())
for key in self.DEFAULT_CONFIG:
setattr(
self, key,
self.config.get('main', {}).get(key, None)
or self.DEFAULT_CONFIG[key])
# Initialize Statsd Client
self.statsd = DogStatsd(host=self.host,
port=self.port,
namespace=self.app_key)
self.publish_templates = self.DEFAULT_PUBLISH_TEMPLATES
publish_templates = self.config.get('publish_templates', {})
self.publish_templates.update(publish_templates)
self.counters = self.config.get('counters', {})
self.gauges = self.config.get('gauges', {})
self.timers = self.config.get('timers', [])
self.histograms = self.config.get('histograms', {})
self.sets = self.config.get('sets', {})
self.timers_start_keys = self._get_timers_keys_list('start')
self.timers_end_keys = self._get_timers_keys_list('end')
self.timers_value_keys = self._get_timers_keys_list('value')
def __init__(self,
host: str = 'localhost',
port: int = 8125,
prefix: str = 'faust-app',
rate: float = 1.0,
**kwargs: Any) -> None:
self.client = DogStatsd(host=host,
port=port,
namespace=prefix,
**kwargs)
self.rate = rate
self.sanitize_re = re.compile(r'[^0-9a-zA-Z_]')
self.re_substitution = '_'
def __init__(self,
host='localhost',
port=8125,
prefix=None,
maxudpsize=512,
ipv6=False):
if ipv6:
_log.warning("DogStatsdAdapter() was 'ipv6'. This is ignored")
self._dd_client = DogStatsd(host=host,
port=port,
namespace=prefix,
max_buffer_size=maxudpsize)
def __init__(self,
app,
statsd_host='localhost',
statsd_port='8125',
statsd_prefix='openstack',
statsd_replace='id'):
self.app = app
self.replace_strategy = _ReplaceStrategy(
os.getenv('STATSD_REPLACE', statsd_replace))
self.client = DogStatsd(host=os.getenv('STATSD_HOST', statsd_host),
port=int(os.getenv('STATSD_PORT',
statsd_port)),
namespace=os.getenv('STATSD_PREFIX',
statsd_prefix))
def _get_client(self, host, port, namespace):
return DogStatsd(host=host, port=port, namespace=namespace)
def __init__(
self,
host: str = 'localhost',
port: int = 8125,
socket_path: str = None,
prefix: str = None,
default_tags: Dict[str, Any] = None,
):
"""
This is a wrapper that does primarily this:
- setup connection to statsd server
- wrap measuring methods such that they can be used as various things
(context managers, decorators)
-
:param host: Host of the statsd server
:param port: Port of the statsd server
:param prefix: Default prefix to add to all metrics
:param default_tags: Default tags to add to all metrics
"""
# Setup stats connection
self._statsd = DogStatsd(
host=host,
port=port,
socket_path=socket_path,
constant_tags=_dict_as_statsd_tags(default_tags))
# Add measurement methods
self.increment = self._wrap_measurement_method(
self._statsd.increment,
default_value=1,
prefix=self._join_with_prefix(config.measurement.PREFIX_COUNTER,
prefix),
)
self.decrement = self._wrap_measurement_method(
self._statsd.decrement,
default_value=1,
prefix=self._join_with_prefix(config.measurement.PREFIX_COUNTER,
prefix),
)
self.gauge = self._wrap_measurement_method(
self._statsd.gauge,
prefix=self._join_with_prefix(config.measurement.PREFIX_GAUGE,
prefix))
self.histogram = self._wrap_measurement_method(
self._statsd.histogram,
prefix=self._join_with_prefix(config.measurement.PREFIX_HISTOGRAM,
prefix))
self.timing = self._wrap_measurement_method(
self._statsd.timing,
prefix=self._join_with_prefix(config.measurement.PREFIX_TIMING,
prefix))
self.set = self._wrap_measurement_method(
self._statsd.set,
prefix=self._join_with_prefix(config.measurement.PREFIX_SET,
prefix))
# Our own augmented measurement primitives
self.timer = self._wrap_measurement_method(
self._statsd.timing,
prefix=self._join_with_prefix(config.measurement.PREFIX_TIMING,
prefix),
wrapper=TimerMeasuringPrimitive,
)
self.counter = self._wrap_measurement_method(
self._statsd.increment,
prefix=self._join_with_prefix(config.measurement.PREFIX_COUNTER,
prefix),
wrapper=CounterMeasuringPrimitive,
)
def __init__(self, app, config, logger=logging.getLogger(__name__)):
self.logger = logger
self.app = app
self.wsgi_config = config
self.watcher_config = {}
self.cadf_service_name = self.wsgi_config.get('cadf_service_name',
None)
self.service_type = self.wsgi_config.get('service_type',
taxonomy.UNKNOWN)
# get the project uid from the request path or from the token (default)
self.is_project_id_from_path = common.string_to_bool(
self.wsgi_config.get('target_project_id_from_path', 'False'))
# get the project id from the service catalog (see documentation on keystone auth_token middleware)
self.is_project_id_from_service_catalog = common.string_to_bool(
self.wsgi_config.get('target_project_id_from_service_catalog',
'False'))
# whether to include the target project id in the metrics
self.is_include_target_project_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_target_project_id_in_metric',
'True'))
# whether to include the target domain id in the metrics
self.is_include_target_domain_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_target_domain_id_in_metric', 'True'))
# whether to include the initiator user id in the metrics
self.is_include_initiator_user_id_in_metric = common.string_to_bool(
self.wsgi_config.get('include_initiator_user_id_in_metric',
'False'))
config_file_path = config.get('config_file', None)
if config_file_path:
try:
self.watcher_config = load_config(config_file_path)
except errors.ConfigError as e:
self.logger.debug("custom actions not available: %s", str(e))
custom_action_config = self.watcher_config.get('custom_actions', {})
path_keywords = self.watcher_config.get('path_keywords', {})
keyword_exclusions = self.watcher_config.get('keyword_exclusions', {})
regex_mapping = self.watcher_config.get('regex_path_mapping', {})
# init the strategy used to determine the target type uri
strat = STRATEGIES.get(self.service_type, strategies.BaseCADFStrategy)
# set custom prefix to target type URI or use defaults
target_type_uri_prefix = common.SERVICE_TYPE_CADF_PREFIX_MAP.get(
self.service_type, 'service/{0}'.format(self.service_type))
if self.cadf_service_name:
target_type_uri_prefix = self.cadf_service_name
strategy = strat(target_type_uri_prefix=target_type_uri_prefix,
path_keywords=path_keywords,
keyword_exclusions=keyword_exclusions,
custom_action_config=custom_action_config,
regex_mapping=regex_mapping)
self.strategy = strategy
self.metric_client = DogStatsd(
host=self.wsgi_config.get("statsd_host", "127.0.0.1"),
port=int(self.wsgi_config.get("statsd_port", 9125)),
namespace=self.wsgi_config.get("statsd_namespace",
"openstack_watcher"))
def __init__(self, app, **conf):
self.app = app
# Configuration via paste.ini.
self.__conf = conf
self.logger = log.Logger(conf.get('log_name', __name__))
# StatsD is used to emit metrics.
statsd_host = self.__conf.get('statsd_host', '127.0.0.1')
statsd_port = common.to_int(self.__conf.get('statsd_port', 9125))
statsd_prefix = self.__conf.get('statsd_prefix',
common.Constants.metric_prefix)
# Init StatsD client.
self.metricsClient = DogStatsd(
host=os.getenv('STATSD_HOST', statsd_host),
port=int(os.getenv('STATSD_PORT', statsd_port)),
namespace=os.getenv('STATSD_PREFIX', statsd_prefix))
# Get backend configuration.
# Backend is used to store count of requests.
self.backend_host = self.__conf.get('backend_host', '127.0.0.1')
self.backend_port = common.to_int(self.__conf.get('backend_port'),
6379)
self.logger.debug("using backend '{0}' on '{1}:{2}'".format(
'redis', self.backend_host, self.backend_port))
backend_timeout_seconds = common.to_int(
self.__conf.get('backend_timeout_seconds'), 20)
backend_max_connections = common.to_int(
self.__conf.get('backend_max_connections'), 100)
# Load configuration file.
self.config = {}
config_file = self.__conf.get('config_file', None)
if config_file:
try:
self.config = common.load_config(config_file)
except errors.ConfigError as e:
self.logger.warning("error loading configuration: {0}".format(
str(e)))
self.service_type = self.__conf.get('service_type', None)
# This is required to trim the prefix from the target_type_uri.
# Example:
# service_type = identity
# cadf_service_name = data/security
# target_type_uri = data/security/auth/tokens -> auth/tokens
self.cadf_service_name = self.__conf.get('cadf_service_name', None)
if common.is_none_or_unknown(self.cadf_service_name):
self.cadf_service_name = common.CADF_SERVICE_TYPE_PREFIX_MAP.get(
self.service_type, None)
# Use configured parameters or ensure defaults.
max_sleep_time_seconds = common.to_int(
self.__conf.get(common.Constants.max_sleep_time_seconds), 20)
log_sleep_time_seconds = common.to_int(
self.__conf.get(common.Constants.log_sleep_time_seconds), 10)
# Setup ratelimit and blacklist response.
self._setup_response()
# White-/blacklist can contain project, domain, user ids or the client ip address.
# Don't apply rate limits to localhost.
default_whitelist = ['127.0.0.1', 'localhost']
config_whitelist = self.config.get('whitelist', [])
self.whitelist = default_whitelist + config_whitelist
self.whitelist_users = self.config.get('whitelist_users', [])
self.blacklist = self.config.get('blacklist', [])
self.blacklist_users = self.config.get('blacklist_users', [])
# Mapping of potentially multiple CADF actions to one action.
self.rate_limit_groups = self.config.get('groups', {})
# Configurable scope in which a rate limit is applied. Defaults to initiator project id.
# Rate limits are applied based on the tuple of (rate_limit_by, action, target_type_uri).
self.rate_limit_by = self.__conf.get(
'rate_limit_by', common.Constants.initiator_project_id)
# Accuracy of the request timestamps used. Defaults to nanosecond accuracy.
clock_accuracy = int(
1 / units.Units.parse(self.__conf.get('clock_accuracy', '1ns')))
self.backend = rate_limit_backend.RedisBackend(
host=self.backend_host,
port=self.backend_port,
rate_limit_response=self.ratelimit_response,
max_sleep_time_seconds=max_sleep_time_seconds,
log_sleep_time_seconds=log_sleep_time_seconds,
timeout_seconds=backend_timeout_seconds,
max_connections=backend_max_connections,
clock_accuracy=clock_accuracy,
)
# Test if the backend is ready.
is_available, msg = self.backend.is_available()
if not is_available:
self.logger.warning(
"rate limit not possible. the backend is not available: {0}".
format(msg))
# Provider for rate limits. Defaults to configuration file.
# Also supports Limes.
configuration_ratelimit_provider = provider.ConfigurationRateLimitProvider(
service_type=self.service_type)
# Force load of rate limits from configuration file.
configuration_ratelimit_provider.read_rate_limits_from_config(
config_file)
self.ratelimit_provider = configuration_ratelimit_provider
# If limes is enabled and we want to rate limit by initiator|target project id,
# Set LimesRateLimitProvider as the provider for rate limits.
limes_enabled = self.__conf.get('limes_enabled', False)
if limes_enabled:
self.__setup_limes_ratelimit_provider()
self.logger.info("OpenStack Rate Limit Middleware ready for requests.")
pass
def __exit__(self, type, value, traceback):
pass
def __call__(self, func):
def wrapped(*args, **kwargs):
return func(*args, **kwargs)
return wrapped
def timed(*args, **kwargs):
return WithDecorator()
stats.timed = timed
else:
from datadog.dogstatsd import DogStatsd
stats = DogStatsd(host=os.getenv('STATSD_HOST', 'localhost'),
port=int(os.getenv('STATSD_PORT', 9125)),
namespace=os.getenv('STATSD_PREFIX', 'openstack')
)
##
# oslo.vmware.vim_util
def _get_token(retrieve_result):
"""Get token from result to obtain next set of results.
:retrieve_result: Result of RetrievePropertiesEx API call
:returns: token to obtain next set of results; None if no more results.
"""
return getattr(retrieve_result, 'token', None)
def cancel_retrieval(si, retrieve_result):
def _create_statsd(*args, **kwargs):
# testing mock point
return DogStatsd(*args, **kwargs)
def __init__(self, config):
super().__init__(config)
self.client = DogStatsd()
import os
from datadog import initialize
from datadog.dogstatsd import DogStatsd
import config
root = os.path.dirname(os.path.abspath(__file__))
conf = config.Config(os.path.join(root, "config.ini"))
dogstatsd = DogStatsd(host='localhost',
port=conf.datadog_dogstatsd_port,
constant_tags=conf.datadog_tags)
def setup(self):
from datadog.dogstatsd import DogStatsd
self.client = DogStatsd(host=self.host, port=self.port)
