def __init__(self, app_prefix, dogstatsd=None, dogstatsd_kwargs=None):
if dogstatsd_kwargs is None:
dogstatsd_kwargs = {}
if dogstatsd:
self.dogstatsd = dogstatsd
else:
self.dogstatsd = DogStatsd(**dogstatsd_kwargs)
self.app_prefix = app_prefix
class Stats:
MESSAGE_FLUSHED_METRIC = "message_flushed"
TASK_EXECUTED_TIME_METRIC = "task_executed"
def __init__(self, configuration: Configuration) -> None:
jsonschema.validate(
configuration,
{
"type": "object",
"properties": {
"host": {"type": "string"},
"port": {"type": "integer"},
"message_sampling_rate": {"type": "number", "minimum": 0.0, "maximum": 1.0},
"task_sampling_rate": {"type": "number", "minimum": 0.0, "maximum": 1.0},
},
"required": ["host", "port"],
},
)
self.__dogstatsd = DogStatsd(
host=configuration["host"],
port=configuration["port"],
namespace=METRIC_PREFIX,
)
self.__message_sampling_rate: float = configuration.get("message_sampling_rate", 1.0)
self.__task_sampling_rate: float = configuration.get("task_sampling_rate", 1.0)
def message_flushed(self, start: float) -> None:
self.__record_simple_interval(
start, self.MESSAGE_FLUSHED_METRIC, self.__message_sampling_rate
)
def task_executed(self, start: float, tasktype: str) -> None:
tag = "%s:%s" % ("tasktype", tasktype)
self.__record_simple_interval(
start, self.TASK_EXECUTED_TIME_METRIC, self.__task_sampling_rate, [tag]
)
def __record_simple_interval(
self, start: float, metric: str, sample_rate: float, tags: list = None
) -> None:
now = time.time()
duration = int((now - start) * 1000)
try:
self.__dogstatsd.timing(
metric, duration, tags=tags, sample_rate=sample_rate
)
except Exception as e:
logger.exception(e)
def __init__(self):
""" Virtually private constructor. """
if Singleton.__statsd != None:
raise Exception("This class is a singleton!")
else:
Singleton.__statsd = DogStatsd(host=os.environ['NODE_IP'],
port=9125)
def init_plugins():
"""初始化日志、错误追踪、打点插件"""
from everyclass.rpc import init as init_rpc
from everyclass.common.flask import print_config
# Sentry
if plugin_available("sentry"):
sentry.init_app(app=__app)
sentry_handler = SentryHandler(sentry.client)
sentry_handler.setLevel(logging.WARNING)
logging.getLogger().addHandler(sentry_handler)
init_rpc(sentry=sentry)
logger.info('Sentry is inited because you are in {} mode.'.format(
__app.config['CONFIG_NAME']))
# metrics
global statsd
statsd = DogStatsd(
namespace=
f"{__app.config['SERVICE_NAME']}.{os.environ.get('MODE').lower()}",
use_default_route=True)
init_rpc(logger=logger)
print_config(__app, logger)
def __init__(self, host, port, namespace):
if not StatsD.instance:
StatsD.instance = StatsD.__StatsD(host, port, namespace)
else:
StatsD.instance.client = DogStatsd(host=host,
port=port,
namespace=namespace)
def create_metrics(prefix: str, tags: Optional[Tags] = None) -> MetricsBackend:
"""Create a DogStatsd object if DOGSTATSD_HOST and DOGSTATSD_PORT are defined,
with the specified prefix and tags. Return a DummyMetricsBackend otherwise.
Prefixes must start with `snuba.<category>`, for example: `snuba.processor`.
"""
bits = prefix.split(".", 2)
assert (len(bits) >= 2
and bits[0] == "snuba"), "prefix must be like `snuba.<category>`"
host = settings.DOGSTATSD_HOST
port = settings.DOGSTATSD_PORT
if host is None and port is None:
from snuba.utils.metrics.backends.dummy import DummyMetricsBackend
return DummyMetricsBackend()
elif host is None or port is None:
raise ValueError(
f"DOGSTATSD_HOST and DOGSTATSD_PORT should both be None or not None. Found DOGSTATSD_HOST: {host}, DOGSTATSD_PORT: {port} instead."
)
from datadog import DogStatsd
from snuba.utils.metrics.backends.datadog import DatadogMetricsBackend
return DatadogMetricsBackend(
DogStatsd(
host=host,
port=port,
namespace=prefix,
constant_tags=[f"{key}:{value}" for key, value in tags.items()]
if tags is not None else None,
), )
def create_service(cls, context, request):
return cls(
DogStatsd(
host=request.registry.settings.get("metrics.host", "127.0.0.1"),
port=int(request.registry.settings.get("metrics.port", 8125)),
namespace=request.registry.settings.get("metrics.namespace"),
use_ms=True,
)
)
def get_dogstatsd_logger(self):
from datadog import DogStatsd
dogstatsd = DogStatsd(
host=conf.get('scheduler', 'statsd_host'),
port=conf.getint('scheduler', 'statsd_port'),
namespace=conf.get('scheduler', 'statsd_prefix'),
constant_tags=self.get_constant_tags())
dogstatsd_allow_list = conf.get('scheduler', 'statsd_allow_list', fallback=None)
allow_list_validator = AllowListValidator(dogstatsd_allow_list)
return SafeDogStatsdLogger(dogstatsd, allow_list_validator)
def create_client(self, statsd_config: StatsParamConfig):
from datadog import DogStatsd # noqa
statsd = DogStatsd(
host=statsd_config.statsd_host,
port=statsd_config.statsd_port,
namespace=statsd_config.statsd_prefix,
constant_tags=statsd_config.statsd_prefix,
)
self.statsd = statsd
def create_metrics(host, port, prefix, tags=None):
"""Create a DogStatsd object with the specified prefix and tags. Prefixes
must start with `snuba.<category>`, for example: `snuba.processor`."""
from datadog import DogStatsd
bits = prefix.split('.', 2)
assert len(bits) >= 2 and bits[0] == 'snuba', "prefix must be like `snuba.<category>`"
return DogStatsd(host=host, port=port, namespace=prefix, constant_tags=tags)
def includeme(config):
config.include(".pyramid_datadog")
config.configure_metrics(
DogStatsd(
host=os.environ.get("DATADOG_HOST", "127.0.0.1"),
port=int(os.environ.get("DATADOG_PORT", 8125)),
namespace=os.environ.get("DATADOG_NAMESPACE"),
use_ms=True,
use_default_route=bool(
os.environ.get("DATADOG_USE_DEFAULT_ROUTE")),
))
def get_dogstatsd_logger(cls):
"""Get DataDog statsd logger"""
from datadog import DogStatsd
dogstatsd = DogStatsd(
host=conf.get('metrics', 'statsd_host'),
port=conf.getint('metrics', 'statsd_port'),
namespace=conf.get('metrics', 'statsd_prefix'),
constant_tags=cls.get_constant_tags(),
)
dogstatsd_allow_list = conf.get('metrics', 'statsd_allow_list', fallback=None)
allow_list_validator = AllowListValidator(dogstatsd_allow_list)
return SafeDogStatsdLogger(dogstatsd, allow_list_validator)
def __init__(self, configuration: Configuration) -> None:
jsonschema.validate(
configuration,
{
"type": "object",
"properties": {
"host": {"type": "string"},
"port": {"type": "integer"},
"message_sampling_rate": {"type": "number", "minimum": 0.0, "maximum": 1.0},
"task_sampling_rate": {"type": "number", "minimum": 0.0, "maximum": 1.0},
},
"required": ["host", "port"],
},
)
self.__dogstatsd = DogStatsd(
host=configuration["host"],
port=configuration["port"],
namespace=METRIC_PREFIX,
)
self.__message_sampling_rate: float = configuration.get("message_sampling_rate", 1.0)
self.__task_sampling_rate: float = configuration.get("task_sampling_rate", 1.0)
def get_datadog_statsd(config=None):
"""
Create DataDog statsd client
:type config: pypipes.config.Config
:return: statsd client
:rtype: datadog.DogStatsd
"""
global datadog_statsd
if datadog_statsd is None:
from datadog import DogStatsd
if config:
host = config.datadog.get('host', 'localhost')
port = config.datadog.get('port', 8125)
global_tags = config.datadog.get('tags', [])
namespace = config.datadog.get('namespace')
# DogStatsd client sends metrics to DataDog agent via UDP, like statsd client.
datadog_statsd = DogStatsd(host, port,
namespace=namespace,
constant_tags=global_tags)
else:
# use default
datadog_statsd = DogStatsd()
return datadog_statsd
class DatadogMiddleware:
def __init__(self, app_prefix, dogstatsd_kwargs=None):
if dogstatsd_kwargs is None:
dogstatsd_kwargs = {}
self.dogstatsd = DogStatsd(**dogstatsd_kwargs)
self.app_prefix = app_prefix
async def __call__(self, request, handler):
with self.dogstatsd.timed("{0}.request.time".format(self.app_prefix),
tags=self.get_tags(request)):
return await handler(request)
def get_tags(self, request):
return [
"http_method:{0}".format(request.method),
"http_host:{0}".format(request.host),
"http_path:{0}".format(request.path),
]
def create_metrics(host: str,
port: int,
prefix: str,
tags: Optional[Tags] = None) -> MetricsBackend:
"""Create a DogStatsd object with the specified prefix and tags. Prefixes
must start with `snuba.<category>`, for example: `snuba.processor`."""
from datadog import DogStatsd
from snuba.utils.metrics.backends.datadog import DatadogMetricsBackend
bits = prefix.split('.', 2)
assert len(bits) >= 2 and bits[
0] == 'snuba', "prefix must be like `snuba.<category>`"
return DatadogMetricsBackend(
DogStatsd(
host=host,
port=port,
namespace=prefix,
constant_tags=[f'{key}:{value}' for key, value in tags.items()]
if tags is not None else None,
), )
class DatadogMiddleware:
def __init__(self, app_prefix, dogstatsd_kwargs=None):
if dogstatsd_kwargs is None:
dogstatsd_kwargs = {}
self.dogstatsd = DogStatsd(**dogstatsd_kwargs)
self.app_prefix = app_prefix
async def __call__(self, app, handler):
return partial(self.middleware, handler)
async def middleware(self, handler, request):
tags = [
'http_method:{0}'.format(request.method),
'http_version:{0}'.format(request.version),
'http_host:{0}'.format(request.host),
'http_path:{0}'.format(request.path),
'request_type:{0}'.format(request.GET.getone('type', None)),
]
with self.dogstatsd.timed('{0}.request.time'.format(self.app_prefix), tags=tags):
return await handler(request)
class StatsdMetricPlugin(MetricPlugin):
title = "Statsd"
slug = "statsd-metrics"
description = "Adds support for sending metrics to Statsd"
version = plug.VERSION
def __init__(self):
host = current_app.config.get("STATSD_HOST")
port = current_app.config.get("STATSD_PORT")
prefix = current_app.config.get("STATSD_PREFIX")
self.statsd = DogStatsd(host=host, port=port, namespace=prefix)
def submit(self,
metric_name,
metric_type,
metric_value,
metric_tags=None,
options=None):
valid_types = ["COUNTER", "GAUGE", "TIMER"]
tags = []
if metric_type.upper() not in valid_types:
raise Exception(
"Invalid Metric Type for Statsd, '{metric}' choose from: {options}"
.format(metric=metric_type, options=",".join(valid_types)))
if metric_tags:
if not isinstance(metric_tags, dict):
raise Exception(
"Invalid Metric Tags for Statsd: Tags must be in dict format"
)
else:
tags = map(lambda e: "{0}:{1}".format(*e), metric_tags.items())
if metric_type.upper() == "COUNTER":
self.statsd.increment(metric_name, metric_value, tags)
elif metric_type.upper() == "GAUGE":
self.statsd.gauge(metric_name, metric_value, tags)
elif metric_type.upper() == "TIMER":
self.statsd.timing(metric_name, metric_value, tags)
return
class StatsdMetricPlugin(MetricPlugin):
title = 'Statsd'
slug = 'statsd-metrics'
description = 'Adds support for sending metrics to Statsd'
version = plug.VERSION
def __init__(self):
host = current_app.config.get('STATSD_HOST')
port = current_app.config.get('STATSD_PORT')
prefix = current_app.config.get('STATSD_PREFIX')
self.statsd = DogStatsd(host=host, port=port, namespace=prefix)
def submit(self, metric_name, metric_type, metric_value, metric_tags=None, options=None):
valid_types = ['COUNTER', 'GAUGE', 'TIMER']
tags = []
if metric_type.upper() not in valid_types:
raise Exception(
"Invalid Metric Type for Statsd, '{metric}' choose from: {options}".format(
metric=metric_type, options=','.join(valid_types)
)
)
if metric_tags:
if not isinstance(metric_tags, dict):
raise Exception("Invalid Metric Tags for Statsd: Tags must be in dict format")
else:
tags = map(lambda e: "{0}:{1}".format(*e), metric_tags.items())
if metric_type.upper() == 'COUNTER':
self.statsd.increment(metric_name, metric_value, tags)
elif metric_type.upper() == 'GAUGE':
self.statsd.gauge(metric_name, metric_value, tags)
elif metric_type.upper() == 'TIMER':
self.statsd.timing(metric_name, metric_value, tags)
return
import os
import time
from datadog import DogStatsd
from django.utils.deprecation import MiddlewareMixin
statsd_host = os.environ.get("STATSD_ENDPOINT")
statsd_port = int(os.environ.get("STATSD_PORT"))
statsd = DogStatsd(host=statsd_host, port=statsd_port)
REQUEST_LATENCY_METRIC_NAME = "django_request_latency_seconds"
REQUEST_COUNT_METRIC_NAME = "django_request_count"
class StatsdMetricsMiddleware(MiddlewareMixin):
def process_request(self, request):
request.start_time = time.time()
def get_view_name(self, request):
view_name = "<unnamed view>"
if hasattr(request, "resolver_match"):
if request.resolver_match is not None:
if request.resolver_match.view_name is not None:
view_name = request.resolver_match.view_name
return view_name
def process_response(self, request, response):
statsd.increment(
REQUEST_COUNT_METRIC_NAME,
tags=[
"service:django_worker",
def __init__(self):
host = current_app.config.get('STATSD_HOST')
port = current_app.config.get('STATSD_PORT')
prefix = current_app.config.get('STATSD_PREFIX')
self.statsd = DogStatsd(host=host, port=port, namespace=prefix)
from django.conf import settings
from datadog import DogStatsd
_STATSD = DogStatsd(host=settings.DD_STATSD_ADDR, port=settings.DD_STATSD_PORT,
namespace=settings.DD_STATSD_NAMESPACE)
def _prepare_tags(tags):
"""
>>> _prepare_tags({'protocol': 'http'})
['protocol:http']
"""
if not tags:
return []
return [f'{k}:{v}' for k, v in tags.items()]
def gauge(metric, value, tags=None):
"""
Record the value of a gauge, optionally setting a list of tags and a
sample rate.
>>> gauge('users.online', 123)
>>> gauge('active.connections', 1001, tags={'protocol': 'http'})
"""
_STATSD.gauge(metric, value=value, tags=_prepare_tags(tags))
def increment(metric, value=1, tags=None):
"""
Increment a counter, optionally setting a value, tags and a sample
from flask import request
from datadog import DogStatsd
import time
import sys
statsd = DogStatsd(host="statsd", port=9125)
REQUEST_LATENCY_METRIC_NAME = 'request_latency_seconds'
REQUEST_COUNT_METRIC_NAME = 'request_count'
def start_timer():
request.start_time = time.time()
def stop_timer(response):
resp_time = time.time() - request.start_time
statsd.histogram(REQUEST_LATENCY_METRIC_NAME,
resp_time,
tags=[
'service:webapp',
'endpoint:%s' % request.path,
])
return response
def record_request_data(response):
statsd.increment(REQUEST_COUNT_METRIC_NAME,
tags=[
'service:webapp',
'method:%s' % request.method,
'endpoint:%s' % request.path,
import logging
from json import JSONDecodeError
from flask import Flask, request, abort, json
from functools import wraps
from flask_api import status
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.exc import IntegrityError
from wtforms import Form, StringField, validators
from wtforms.validators import (DataRequired, Email)
from flask_cors import CORS
from datadog import DogStatsd
import config
statsd = DogStatsd(config.STATSD_HOST, config.STATSD_PORT, namespace='app-registration')
app = Flask(__name__)
app.API_KEY = config.API_KEY
CORS(app)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://{}:{}@{}:5432/{}'.format(config.DB_ROLE,
config.DB_PASSWORD,
config.DB_HOST,
config.DB_NAME.lower())
app.config['APP_ID'] = config.API_KEY
db = SQLAlchemy(app)
APP_ID_LENGTH = 4
MIN_LENGTH = 2
MAX_LENGTH = 250
def __init__(self):
host = current_app.config.get("STATSD_HOST")
port = current_app.config.get("STATSD_PORT")
prefix = current_app.config.get("STATSD_PREFIX")
self.statsd = DogStatsd(host=host, port=port, namespace=prefix)
import time
from datadog import DogStatsd
import time
import sys
from django.db.models.signals import post_save
from django.db.backends.signals import connection_created
from django.contrib.auth.signals import user_logged_in, user_logged_out, user_login_failed
from django.dispatch import receiver
from django.core.cache.backends import locmem
statsd = DogStatsd(host="statsd", use_ms=True, port=9125)
REQUEST_LATENCY_METRIC_NAME = 'django_request_latency_ms'
REQUEST_COUNT_METRIC_NAME = 'django_request_count'
DJANGO_EXCEPTION_COUNTER = 'django_exceptions'
DJANGO_MODELS_NEW_ROW_METRIC_NAME = 'django_models_create_count'
DJANGO_MODELS_UPDATE_ROW_METRIC_NAME = 'django_models_update_count'
DJANGO_DB_CONNECTIONS_CREATED_METRIC_NAME = 'django_database_connections_count'
DJANGO_USER_LOGIN_METRIC_NAME = 'django_user_login'
DJANGO_LOCAL_CACHE_METRIC_NAME = 'django_cache_localmem'
@receiver(post_save)
def update_models_save_counter(sender, instance, created, raw, using, update_fields, **kwargs):
if created:
statsd.increment(DJANGO_MODELS_NEW_ROW_METRIC_NAME,
tags=[
'model:%s' % sender,
'using:%s' % using,
]
return json.JSONEncoder.default(self, obj)
id_descriptions = [line.rstrip('\n') for line in open('model_output.txt')]
classifiers = json.load(open('gender_classifier_output.json'))
rnd_i = random.randrange(0, len(id_descriptions))
config = Config(metrics_host=(os.getenv('METRICS_HOSTNAME', 'localhost')),
env_name=os.environ['ENV_NAME'],
hostname=os.environ['HOSTNAME'],
identity=Identity(
description=id_descriptions[rnd_i],
femaleness=classifiers[rnd_i][0]['classification'][0]['p'],
maleness=classifiers[rnd_i][0]['classification'][1]['p'],
))
statsd = DogStatsd(host=config.metrics_host)
statsd.constant_tags = [
f'sex:{config.identity.sex.name}',
f'identity:{config.identity.description}',
f'hostname:{config.hostname}',
f'env_name:{config.env_name}',
]
app = Flask(__name__)
@app.route('/config', methods=['GET'])
def get_config():
response = Response(json.dumps(config, cls=ConfigEncoder, indent=2),
status=200,
mimetype='application/json')
# Inject request id to a request's context
RequestID(app, request_id_parser=amazon_elb_trace_id, request_id_generator=req_id_generator)
# Setup logging
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(asctime)s | level=%(levelname)s | request_id=%(request_id)s | %(message)s'))
handler.addFilter(RequestIDLogFilter())
logging.root.addHandler(handler)
logger = logging.getLogger('migration')
logger.setLevel('INFO')
kin_logger = logging.getLogger('kin')
kin_logger.setLevel('ERROR')
# Setup DogStatsd
statsd = DogStatsd(host=config.STATSD_HOST, port=config.STATSD_PORT, namespace='migration')
# Setup kin
# Passphrase is not needed for the old environment since we don't send any txs to it
old_env = kin.Environment('OLD', config.OLD_HORIZON, '')
new_env = kin.Environment('NEW', config.NEW_HORIZON, config.NEW_PASSPHRASE)
old_client = kin.KinClient(old_env)
new_client = kin.KinClient(new_env)
channels = create_channels(config.MAIN_SEED, new_env, config.CHANNEL_COUNT, 0, config.CHANNEL_SALT)
main_account = new_client.kin_account(config.MAIN_SEED, channels, app_id=config.APP_ID)
logger.info(f'Initialized app with address: {main_account.keypair.public_address}, '
f'Old horizon: {config.OLD_HORIZON}, '
f'New horizon: {config.NEW_HORIZON}')
from datadog import DogStatsd
from django.conf import settings
try:
from django.urls import resolve, Resolver404
except ImportError:
from django.core.urlresolvers import resolve, Resolver404
try:
from django.utils.deprecation import MiddlewareMixin
base_class = MiddlewareMixin
except ImportError:
base_class = object
statsd_host = getattr(settings, 'FDJANGODOG_STATSD_HOST', 'localhost')
statsd = DogStatsd(host=statsd_host)
class FDjangoDogMiddleware(base_class):
APP_NAME = settings.FDJANGODOG_APP_NAME
DD_TIMING_ATTRIBUTE = '_dd_start_time'
def __init__(self, *args, **kwargs):
super(FDjangoDogMiddleware, self).__init__(*args, **kwargs)
self.stats = statsd
self.timing_metric = '{}.request_time'.format(self.APP_NAME)
def _get_elapsed_time(self, request):
return time.time() - getattr(request, self.DD_TIMING_ATTRIBUTE)
import os
from datadog import initialize, DogStatsd
statsd = DogStatsd(max_buffer_size=1)
options = {
'api_key': os.environ['DATADOG_API_KEY'],
'app_key': os.environ['DATADOG_APP_KEY']
}
initialize(**options)
def get_statsd():
"""Return statsd client."""
return DogStatsd(host=Config().HOST)
import time
from datadog import DogStatsd
from django.conf import settings
try:
from django.utils.deprecation import MiddlewareMixin
except ImportError:
class MiddlewareMixin(object):
pass
statsd = DogStatsd(host=getattr(settings, 'STATSD_HOST', "localhost"),
port=getattr(settings, 'STATSD_PORT', 8125),
namespace=getattr(settings, 'STATSD_PREFIX', None))
class RequestLatencyMiddleware(MiddlewareMixin):
"""The middleware measures time in seconds spent serving HTTP requests.
Put the middleware to the top in the MIDDLEWARE_CLASSES, so it runs before
all middlewares on request phase and last on response phase.
Do not use labels to store dimensions with high cardinality, e.g.,
/v1/books/<book-id> path.
"""
def process_request(self, request):
request._begun_at = time.time()
