def __init__(self, app_prefix, dogstatsd=None, dogstatsd_kwargs=None): if dogstatsd_kwargs is None: dogstatsd_kwargs = {} if dogstatsd: self.dogstatsd = dogstatsd else: self.dogstatsd = DogStatsd(**dogstatsd_kwargs) self.app_prefix = app_prefix
class Stats: MESSAGE_FLUSHED_METRIC = "message_flushed" TASK_EXECUTED_TIME_METRIC = "task_executed" def __init__(self, configuration: Configuration) -> None: jsonschema.validate( configuration, { "type": "object", "properties": { "host": {"type": "string"}, "port": {"type": "integer"}, "message_sampling_rate": {"type": "number", "minimum": 0.0, "maximum": 1.0}, "task_sampling_rate": {"type": "number", "minimum": 0.0, "maximum": 1.0}, }, "required": ["host", "port"], }, ) self.__dogstatsd = DogStatsd( host=configuration["host"], port=configuration["port"], namespace=METRIC_PREFIX, ) self.__message_sampling_rate: float = configuration.get("message_sampling_rate", 1.0) self.__task_sampling_rate: float = configuration.get("task_sampling_rate", 1.0) def message_flushed(self, start: float) -> None: self.__record_simple_interval( start, self.MESSAGE_FLUSHED_METRIC, self.__message_sampling_rate ) def task_executed(self, start: float, tasktype: str) -> None: tag = "%s:%s" % ("tasktype", tasktype) self.__record_simple_interval( start, self.TASK_EXECUTED_TIME_METRIC, self.__task_sampling_rate, [tag] ) def __record_simple_interval( self, start: float, metric: str, sample_rate: float, tags: list = None ) -> None: now = time.time() duration = int((now - start) * 1000) try: self.__dogstatsd.timing( metric, duration, tags=tags, sample_rate=sample_rate ) except Exception as e: logger.exception(e)
def __init__(self): """ Virtually private constructor. """ if Singleton.__statsd != None: raise Exception("This class is a singleton!") else: Singleton.__statsd = DogStatsd(host=os.environ['NODE_IP'], port=9125)
def init_plugins(): """初始化日志、错误追踪、打点插件""" from everyclass.rpc import init as init_rpc from everyclass.common.flask import print_config # Sentry if plugin_available("sentry"): sentry.init_app(app=__app) sentry_handler = SentryHandler(sentry.client) sentry_handler.setLevel(logging.WARNING) logging.getLogger().addHandler(sentry_handler) init_rpc(sentry=sentry) logger.info('Sentry is inited because you are in {} mode.'.format( __app.config['CONFIG_NAME'])) # metrics global statsd statsd = DogStatsd( namespace= f"{__app.config['SERVICE_NAME']}.{os.environ.get('MODE').lower()}", use_default_route=True) init_rpc(logger=logger) print_config(__app, logger)
def __init__(self, host, port, namespace): if not StatsD.instance: StatsD.instance = StatsD.__StatsD(host, port, namespace) else: StatsD.instance.client = DogStatsd(host=host, port=port, namespace=namespace)
def create_metrics(prefix: str, tags: Optional[Tags] = None) -> MetricsBackend: """Create a DogStatsd object if DOGSTATSD_HOST and DOGSTATSD_PORT are defined, with the specified prefix and tags. Return a DummyMetricsBackend otherwise. Prefixes must start with `snuba.<category>`, for example: `snuba.processor`. """ bits = prefix.split(".", 2) assert (len(bits) >= 2 and bits[0] == "snuba"), "prefix must be like `snuba.<category>`" host = settings.DOGSTATSD_HOST port = settings.DOGSTATSD_PORT if host is None and port is None: from snuba.utils.metrics.backends.dummy import DummyMetricsBackend return DummyMetricsBackend() elif host is None or port is None: raise ValueError( f"DOGSTATSD_HOST and DOGSTATSD_PORT should both be None or not None. Found DOGSTATSD_HOST: {host}, DOGSTATSD_PORT: {port} instead." ) from datadog import DogStatsd from snuba.utils.metrics.backends.datadog import DatadogMetricsBackend return DatadogMetricsBackend( DogStatsd( host=host, port=port, namespace=prefix, constant_tags=[f"{key}:{value}" for key, value in tags.items()] if tags is not None else None, ), )
def create_service(cls, context, request): return cls( DogStatsd( host=request.registry.settings.get("metrics.host", "127.0.0.1"), port=int(request.registry.settings.get("metrics.port", 8125)), namespace=request.registry.settings.get("metrics.namespace"), use_ms=True, ) )
def get_dogstatsd_logger(self): from datadog import DogStatsd dogstatsd = DogStatsd( host=conf.get('scheduler', 'statsd_host'), port=conf.getint('scheduler', 'statsd_port'), namespace=conf.get('scheduler', 'statsd_prefix'), constant_tags=self.get_constant_tags()) dogstatsd_allow_list = conf.get('scheduler', 'statsd_allow_list', fallback=None) allow_list_validator = AllowListValidator(dogstatsd_allow_list) return SafeDogStatsdLogger(dogstatsd, allow_list_validator)
def create_client(self, statsd_config: StatsParamConfig): from datadog import DogStatsd # noqa statsd = DogStatsd( host=statsd_config.statsd_host, port=statsd_config.statsd_port, namespace=statsd_config.statsd_prefix, constant_tags=statsd_config.statsd_prefix, ) self.statsd = statsd
def create_metrics(host, port, prefix, tags=None): """Create a DogStatsd object with the specified prefix and tags. Prefixes must start with `snuba.<category>`, for example: `snuba.processor`.""" from datadog import DogStatsd bits = prefix.split('.', 2) assert len(bits) >= 2 and bits[0] == 'snuba', "prefix must be like `snuba.<category>`" return DogStatsd(host=host, port=port, namespace=prefix, constant_tags=tags)
def includeme(config): config.include(".pyramid_datadog") config.configure_metrics( DogStatsd( host=os.environ.get("DATADOG_HOST", "127.0.0.1"), port=int(os.environ.get("DATADOG_PORT", 8125)), namespace=os.environ.get("DATADOG_NAMESPACE"), use_ms=True, use_default_route=bool( os.environ.get("DATADOG_USE_DEFAULT_ROUTE")), ))
def get_dogstatsd_logger(cls): """Get DataDog statsd logger""" from datadog import DogStatsd dogstatsd = DogStatsd( host=conf.get('metrics', 'statsd_host'), port=conf.getint('metrics', 'statsd_port'), namespace=conf.get('metrics', 'statsd_prefix'), constant_tags=cls.get_constant_tags(), ) dogstatsd_allow_list = conf.get('metrics', 'statsd_allow_list', fallback=None) allow_list_validator = AllowListValidator(dogstatsd_allow_list) return SafeDogStatsdLogger(dogstatsd, allow_list_validator)
def __init__(self, configuration: Configuration) -> None: jsonschema.validate( configuration, { "type": "object", "properties": { "host": {"type": "string"}, "port": {"type": "integer"}, "message_sampling_rate": {"type": "number", "minimum": 0.0, "maximum": 1.0}, "task_sampling_rate": {"type": "number", "minimum": 0.0, "maximum": 1.0}, }, "required": ["host", "port"], }, ) self.__dogstatsd = DogStatsd( host=configuration["host"], port=configuration["port"], namespace=METRIC_PREFIX, ) self.__message_sampling_rate: float = configuration.get("message_sampling_rate", 1.0) self.__task_sampling_rate: float = configuration.get("task_sampling_rate", 1.0)
def get_datadog_statsd(config=None): """ Create DataDog statsd client :type config: pypipes.config.Config :return: statsd client :rtype: datadog.DogStatsd """ global datadog_statsd if datadog_statsd is None: from datadog import DogStatsd if config: host = config.datadog.get('host', 'localhost') port = config.datadog.get('port', 8125) global_tags = config.datadog.get('tags', []) namespace = config.datadog.get('namespace') # DogStatsd client sends metrics to DataDog agent via UDP, like statsd client. datadog_statsd = DogStatsd(host, port, namespace=namespace, constant_tags=global_tags) else: # use default datadog_statsd = DogStatsd() return datadog_statsd
class DatadogMiddleware: def __init__(self, app_prefix, dogstatsd_kwargs=None): if dogstatsd_kwargs is None: dogstatsd_kwargs = {} self.dogstatsd = DogStatsd(**dogstatsd_kwargs) self.app_prefix = app_prefix async def __call__(self, request, handler): with self.dogstatsd.timed("{0}.request.time".format(self.app_prefix), tags=self.get_tags(request)): return await handler(request) def get_tags(self, request): return [ "http_method:{0}".format(request.method), "http_host:{0}".format(request.host), "http_path:{0}".format(request.path), ]
def create_metrics(host: str, port: int, prefix: str, tags: Optional[Tags] = None) -> MetricsBackend: """Create a DogStatsd object with the specified prefix and tags. Prefixes must start with `snuba.<category>`, for example: `snuba.processor`.""" from datadog import DogStatsd from snuba.utils.metrics.backends.datadog import DatadogMetricsBackend bits = prefix.split('.', 2) assert len(bits) >= 2 and bits[ 0] == 'snuba', "prefix must be like `snuba.<category>`" return DatadogMetricsBackend( DogStatsd( host=host, port=port, namespace=prefix, constant_tags=[f'{key}:{value}' for key, value in tags.items()] if tags is not None else None, ), )
class DatadogMiddleware: def __init__(self, app_prefix, dogstatsd_kwargs=None): if dogstatsd_kwargs is None: dogstatsd_kwargs = {} self.dogstatsd = DogStatsd(**dogstatsd_kwargs) self.app_prefix = app_prefix async def __call__(self, app, handler): return partial(self.middleware, handler) async def middleware(self, handler, request): tags = [ 'http_method:{0}'.format(request.method), 'http_version:{0}'.format(request.version), 'http_host:{0}'.format(request.host), 'http_path:{0}'.format(request.path), 'request_type:{0}'.format(request.GET.getone('type', None)), ] with self.dogstatsd.timed('{0}.request.time'.format(self.app_prefix), tags=tags): return await handler(request)
class StatsdMetricPlugin(MetricPlugin): title = "Statsd" slug = "statsd-metrics" description = "Adds support for sending metrics to Statsd" version = plug.VERSION def __init__(self): host = current_app.config.get("STATSD_HOST") port = current_app.config.get("STATSD_PORT") prefix = current_app.config.get("STATSD_PREFIX") self.statsd = DogStatsd(host=host, port=port, namespace=prefix) def submit(self, metric_name, metric_type, metric_value, metric_tags=None, options=None): valid_types = ["COUNTER", "GAUGE", "TIMER"] tags = [] if metric_type.upper() not in valid_types: raise Exception( "Invalid Metric Type for Statsd, '{metric}' choose from: {options}" .format(metric=metric_type, options=",".join(valid_types))) if metric_tags: if not isinstance(metric_tags, dict): raise Exception( "Invalid Metric Tags for Statsd: Tags must be in dict format" ) else: tags = map(lambda e: "{0}:{1}".format(*e), metric_tags.items()) if metric_type.upper() == "COUNTER": self.statsd.increment(metric_name, metric_value, tags) elif metric_type.upper() == "GAUGE": self.statsd.gauge(metric_name, metric_value, tags) elif metric_type.upper() == "TIMER": self.statsd.timing(metric_name, metric_value, tags) return
class StatsdMetricPlugin(MetricPlugin): title = 'Statsd' slug = 'statsd-metrics' description = 'Adds support for sending metrics to Statsd' version = plug.VERSION def __init__(self): host = current_app.config.get('STATSD_HOST') port = current_app.config.get('STATSD_PORT') prefix = current_app.config.get('STATSD_PREFIX') self.statsd = DogStatsd(host=host, port=port, namespace=prefix) def submit(self, metric_name, metric_type, metric_value, metric_tags=None, options=None): valid_types = ['COUNTER', 'GAUGE', 'TIMER'] tags = [] if metric_type.upper() not in valid_types: raise Exception( "Invalid Metric Type for Statsd, '{metric}' choose from: {options}".format( metric=metric_type, options=','.join(valid_types) ) ) if metric_tags: if not isinstance(metric_tags, dict): raise Exception("Invalid Metric Tags for Statsd: Tags must be in dict format") else: tags = map(lambda e: "{0}:{1}".format(*e), metric_tags.items()) if metric_type.upper() == 'COUNTER': self.statsd.increment(metric_name, metric_value, tags) elif metric_type.upper() == 'GAUGE': self.statsd.gauge(metric_name, metric_value, tags) elif metric_type.upper() == 'TIMER': self.statsd.timing(metric_name, metric_value, tags) return
import os import time from datadog import DogStatsd from django.utils.deprecation import MiddlewareMixin statsd_host = os.environ.get("STATSD_ENDPOINT") statsd_port = int(os.environ.get("STATSD_PORT")) statsd = DogStatsd(host=statsd_host, port=statsd_port) REQUEST_LATENCY_METRIC_NAME = "django_request_latency_seconds" REQUEST_COUNT_METRIC_NAME = "django_request_count" class StatsdMetricsMiddleware(MiddlewareMixin): def process_request(self, request): request.start_time = time.time() def get_view_name(self, request): view_name = "<unnamed view>" if hasattr(request, "resolver_match"): if request.resolver_match is not None: if request.resolver_match.view_name is not None: view_name = request.resolver_match.view_name return view_name def process_response(self, request, response): statsd.increment( REQUEST_COUNT_METRIC_NAME, tags=[ "service:django_worker",
def __init__(self): host = current_app.config.get('STATSD_HOST') port = current_app.config.get('STATSD_PORT') prefix = current_app.config.get('STATSD_PREFIX') self.statsd = DogStatsd(host=host, port=port, namespace=prefix)
from django.conf import settings from datadog import DogStatsd _STATSD = DogStatsd(host=settings.DD_STATSD_ADDR, port=settings.DD_STATSD_PORT, namespace=settings.DD_STATSD_NAMESPACE) def _prepare_tags(tags): """ >>> _prepare_tags({'protocol': 'http'}) ['protocol:http'] """ if not tags: return [] return [f'{k}:{v}' for k, v in tags.items()] def gauge(metric, value, tags=None): """ Record the value of a gauge, optionally setting a list of tags and a sample rate. >>> gauge('users.online', 123) >>> gauge('active.connections', 1001, tags={'protocol': 'http'}) """ _STATSD.gauge(metric, value=value, tags=_prepare_tags(tags)) def increment(metric, value=1, tags=None): """ Increment a counter, optionally setting a value, tags and a sample
from flask import request from datadog import DogStatsd import time import sys statsd = DogStatsd(host="statsd", port=9125) REQUEST_LATENCY_METRIC_NAME = 'request_latency_seconds' REQUEST_COUNT_METRIC_NAME = 'request_count' def start_timer(): request.start_time = time.time() def stop_timer(response): resp_time = time.time() - request.start_time statsd.histogram(REQUEST_LATENCY_METRIC_NAME, resp_time, tags=[ 'service:webapp', 'endpoint:%s' % request.path, ]) return response def record_request_data(response): statsd.increment(REQUEST_COUNT_METRIC_NAME, tags=[ 'service:webapp', 'method:%s' % request.method, 'endpoint:%s' % request.path,
import logging from json import JSONDecodeError from flask import Flask, request, abort, json from functools import wraps from flask_api import status from flask_sqlalchemy import SQLAlchemy from sqlalchemy.exc import IntegrityError from wtforms import Form, StringField, validators from wtforms.validators import (DataRequired, Email) from flask_cors import CORS from datadog import DogStatsd import config statsd = DogStatsd(config.STATSD_HOST, config.STATSD_PORT, namespace='app-registration') app = Flask(__name__) app.API_KEY = config.API_KEY CORS(app) app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://{}:{}@{}:5432/{}'.format(config.DB_ROLE, config.DB_PASSWORD, config.DB_HOST, config.DB_NAME.lower()) app.config['APP_ID'] = config.API_KEY db = SQLAlchemy(app) APP_ID_LENGTH = 4 MIN_LENGTH = 2 MAX_LENGTH = 250
def __init__(self): host = current_app.config.get("STATSD_HOST") port = current_app.config.get("STATSD_PORT") prefix = current_app.config.get("STATSD_PREFIX") self.statsd = DogStatsd(host=host, port=port, namespace=prefix)
import time from datadog import DogStatsd import time import sys from django.db.models.signals import post_save from django.db.backends.signals import connection_created from django.contrib.auth.signals import user_logged_in, user_logged_out, user_login_failed from django.dispatch import receiver from django.core.cache.backends import locmem statsd = DogStatsd(host="statsd", use_ms=True, port=9125) REQUEST_LATENCY_METRIC_NAME = 'django_request_latency_ms' REQUEST_COUNT_METRIC_NAME = 'django_request_count' DJANGO_EXCEPTION_COUNTER = 'django_exceptions' DJANGO_MODELS_NEW_ROW_METRIC_NAME = 'django_models_create_count' DJANGO_MODELS_UPDATE_ROW_METRIC_NAME = 'django_models_update_count' DJANGO_DB_CONNECTIONS_CREATED_METRIC_NAME = 'django_database_connections_count' DJANGO_USER_LOGIN_METRIC_NAME = 'django_user_login' DJANGO_LOCAL_CACHE_METRIC_NAME = 'django_cache_localmem' @receiver(post_save) def update_models_save_counter(sender, instance, created, raw, using, update_fields, **kwargs): if created: statsd.increment(DJANGO_MODELS_NEW_ROW_METRIC_NAME, tags=[ 'model:%s' % sender, 'using:%s' % using, ]
return json.JSONEncoder.default(self, obj) id_descriptions = [line.rstrip('\n') for line in open('model_output.txt')] classifiers = json.load(open('gender_classifier_output.json')) rnd_i = random.randrange(0, len(id_descriptions)) config = Config(metrics_host=(os.getenv('METRICS_HOSTNAME', 'localhost')), env_name=os.environ['ENV_NAME'], hostname=os.environ['HOSTNAME'], identity=Identity( description=id_descriptions[rnd_i], femaleness=classifiers[rnd_i][0]['classification'][0]['p'], maleness=classifiers[rnd_i][0]['classification'][1]['p'], )) statsd = DogStatsd(host=config.metrics_host) statsd.constant_tags = [ f'sex:{config.identity.sex.name}', f'identity:{config.identity.description}', f'hostname:{config.hostname}', f'env_name:{config.env_name}', ] app = Flask(__name__) @app.route('/config', methods=['GET']) def get_config(): response = Response(json.dumps(config, cls=ConfigEncoder, indent=2), status=200, mimetype='application/json')
# Inject request id to a request's context RequestID(app, request_id_parser=amazon_elb_trace_id, request_id_generator=req_id_generator) # Setup logging handler = logging.StreamHandler() handler.setFormatter(logging.Formatter('%(asctime)s | level=%(levelname)s | request_id=%(request_id)s | %(message)s')) handler.addFilter(RequestIDLogFilter()) logging.root.addHandler(handler) logger = logging.getLogger('migration') logger.setLevel('INFO') kin_logger = logging.getLogger('kin') kin_logger.setLevel('ERROR') # Setup DogStatsd statsd = DogStatsd(host=config.STATSD_HOST, port=config.STATSD_PORT, namespace='migration') # Setup kin # Passphrase is not needed for the old environment since we don't send any txs to it old_env = kin.Environment('OLD', config.OLD_HORIZON, '') new_env = kin.Environment('NEW', config.NEW_HORIZON, config.NEW_PASSPHRASE) old_client = kin.KinClient(old_env) new_client = kin.KinClient(new_env) channels = create_channels(config.MAIN_SEED, new_env, config.CHANNEL_COUNT, 0, config.CHANNEL_SALT) main_account = new_client.kin_account(config.MAIN_SEED, channels, app_id=config.APP_ID) logger.info(f'Initialized app with address: {main_account.keypair.public_address}, ' f'Old horizon: {config.OLD_HORIZON}, ' f'New horizon: {config.NEW_HORIZON}')
from datadog import DogStatsd from django.conf import settings try: from django.urls import resolve, Resolver404 except ImportError: from django.core.urlresolvers import resolve, Resolver404 try: from django.utils.deprecation import MiddlewareMixin base_class = MiddlewareMixin except ImportError: base_class = object statsd_host = getattr(settings, 'FDJANGODOG_STATSD_HOST', 'localhost') statsd = DogStatsd(host=statsd_host) class FDjangoDogMiddleware(base_class): APP_NAME = settings.FDJANGODOG_APP_NAME DD_TIMING_ATTRIBUTE = '_dd_start_time' def __init__(self, *args, **kwargs): super(FDjangoDogMiddleware, self).__init__(*args, **kwargs) self.stats = statsd self.timing_metric = '{}.request_time'.format(self.APP_NAME) def _get_elapsed_time(self, request): return time.time() - getattr(request, self.DD_TIMING_ATTRIBUTE)
import os from datadog import initialize, DogStatsd statsd = DogStatsd(max_buffer_size=1) options = { 'api_key': os.environ['DATADOG_API_KEY'], 'app_key': os.environ['DATADOG_APP_KEY'] } initialize(**options)
def get_statsd(): """Return statsd client.""" return DogStatsd(host=Config().HOST)
import time from datadog import DogStatsd from django.conf import settings try: from django.utils.deprecation import MiddlewareMixin except ImportError: class MiddlewareMixin(object): pass statsd = DogStatsd(host=getattr(settings, 'STATSD_HOST', "localhost"), port=getattr(settings, 'STATSD_PORT', 8125), namespace=getattr(settings, 'STATSD_PREFIX', None)) class RequestLatencyMiddleware(MiddlewareMixin): """The middleware measures time in seconds spent serving HTTP requests. Put the middleware to the top in the MIDDLEWARE_CLASSES, so it runs before all middlewares on request phase and last on response phase. Do not use labels to store dimensions with high cardinality, e.g., /v1/books/<book-id> path. """ def process_request(self, request): request._begun_at = time.time()