class TestDogStatsd(unittest.TestCase):
def setUp(self):
"""
Set up a default Dogstatsd instance and mock the proc filesystem.
"""
#
self.statsd = DogStatsd()
self.statsd.socket = FakeSocket()
# Mock the proc filesystem
route_data = load_fixtures('route')
self._procfs_mock = patch('datadog.util.compat.builtins.open',
mock_open())
self._procfs_mock.__enter__(
).return_value.readlines.return_value = route_data.split("\n")
def tearDown(self):
"""
Unmock the proc filesystem.
"""
self._procfs_mock.__exit__()
def recv(self):
return self.statsd.socket.recv()
def test_initialization(self):
"""
`initialize` overrides `statsd` default instance attributes.
"""
options = {'statsd_host': "myhost", 'statsd_port': 1234}
# Default values
t.assert_equal(statsd.host, "localhost")
t.assert_equal(statsd.port, 8125)
# After initialization
initialize(**options)
t.assert_equal(statsd.host, "myhost")
t.assert_equal(statsd.port, 1234)
# Add namespace
options['statsd_namespace'] = "mynamespace"
initialize(**options)
t.assert_equal(statsd.host, "myhost")
t.assert_equal(statsd.port, 1234)
t.assert_equal(statsd.namespace, "mynamespace")
# Set `statsd` host to the system's default route
initialize(statsd_use_default_route=True, **options)
t.assert_equal(statsd.host, "172.17.0.1")
t.assert_equal(statsd.port, 1234)
# Add UNIX socket
options['statsd_socket_path'] = '/var/run/dogstatsd.sock'
initialize(**options)
t.assert_equal(statsd.socket_path, options['statsd_socket_path'])
t.assert_equal(statsd.host, None)
t.assert_equal(statsd.port, None)
def test_dogstatsd_initialization_with_env_vars(self):
"""
Dogstatsd can retrieve its config from env vars when
not provided in constructor.
"""
# Setup
with preserve_environment_variable('DD_AGENT_HOST'):
os.environ['DD_AGENT_HOST'] = 'myenvvarhost'
with preserve_environment_variable('DD_DOGSTATSD_PORT'):
os.environ['DD_DOGSTATSD_PORT'] = '4321'
statsd = DogStatsd()
# Assert
t.assert_equal(statsd.host, "myenvvarhost")
t.assert_equal(statsd.port, 4321)
def test_default_route(self):
"""
Dogstatsd host can be dynamically set to the default route.
"""
# Setup
statsd = DogStatsd(use_default_route=True)
# Assert
t.assert_equal(statsd.host, "172.17.0.1")
def test_set(self):
self.statsd.set('set', 123)
assert self.recv() == 'set:123|s'
def test_gauge(self):
self.statsd.gauge('gauge', 123.4)
assert self.recv() == 'gauge:123.4|g'
def test_counter(self):
self.statsd.increment('page.views')
t.assert_equal('page.views:1|c', self.recv())
self.statsd.increment('page.views', 11)
t.assert_equal('page.views:11|c', self.recv())
self.statsd.decrement('page.views')
t.assert_equal('page.views:-1|c', self.recv())
self.statsd.decrement('page.views', 12)
t.assert_equal('page.views:-12|c', self.recv())
def test_histogram(self):
self.statsd.histogram('histo', 123.4)
t.assert_equal('histo:123.4|h', self.recv())
def test_tagged_gauge(self):
self.statsd.gauge('gt',
123.4,
tags=['country:china', 'age:45', 'blue'])
t.assert_equal('gt:123.4|g|#country:china,age:45,blue', self.recv())
def test_tagged_counter(self):
self.statsd.increment('ct', tags=[u'country:españa', 'red'])
t.assert_equal(u'ct:1|c|#country:españa,red', self.recv())
def test_tagged_histogram(self):
self.statsd.histogram('h', 1, tags=['red'])
t.assert_equal('h:1|h|#red', self.recv())
def test_sample_rate(self):
self.statsd.increment('c', sample_rate=0)
assert not self.recv()
for i in range(10000):
self.statsd.increment('sampled_counter', sample_rate=0.3)
self.assert_almost_equal(3000, len(self.statsd.socket.payloads), 150)
t.assert_equal('sampled_counter:1|c|@0.3', self.recv())
def test_tags_and_samples(self):
for i in range(100):
self.statsd.gauge('gst', 23, tags=["sampled"], sample_rate=0.9)
def test_tags_and_samples(self):
for i in range(100):
self.statsd.gauge('gst', 23, tags=["sampled"], sample_rate=0.9)
t.assert_equal('gst:23|g|@0.9|#sampled')
def test_timing(self):
self.statsd.timing('t', 123)
t.assert_equal('t:123|ms', self.recv())
def test_event(self):
self.statsd.event('Title',
u'L1\nL2',
priority='low',
date_happened=1375296969)
t.assert_equal(u'_e{5,6}:Title|L1\\nL2|d:1375296969|p:low',
self.recv())
self.statsd.event('Title',
u'♬ †øU †øU ¥ºu T0µ ♪',
aggregation_key='key',
tags=['t1', 't2:v2'])
t.assert_equal(u'_e{5,19}:Title|♬ †øU †øU ¥ºu T0µ ♪|k:key|#t1,t2:v2',
self.recv())
def test_event_constant_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.event('Title',
u'L1\nL2',
priority='low',
date_happened=1375296969)
t.assert_equal(
u'_e{5,6}:Title|L1\\nL2|d:1375296969|p:low|#bar:baz,foo',
self.recv())
self.statsd.event('Title',
u'♬ †øU †øU ¥ºu T0µ ♪',
aggregation_key='key',
tags=['t1', 't2:v2'])
t.assert_equal(
u'_e{5,19}:Title|♬ †øU †øU ¥ºu T0µ ♪|k:key|#t1,t2:v2,bar:baz,foo',
self.recv())
def test_service_check(self):
now = int(time.time())
self.statsd.service_check('my_check.name',
self.statsd.WARNING,
tags=['key1:val1', 'key2:val2'],
timestamp=now,
hostname='i-abcd1234',
message=u"♬ †øU \n†øU ¥ºu|m: T0µ ♪")
t.assert_equal(
u'_sc|my_check.name|{0}|d:{1}|h:i-abcd1234|#key1:val1,key2:val2|m:{2}'
.format(self.statsd.WARNING, now, u"♬ †øU \\n†øU ¥ºu|m\: T0µ ♪"),
self.recv())
def test_service_check_constant_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
now = int(time.time())
self.statsd.service_check('my_check.name',
self.statsd.WARNING,
timestamp=now,
hostname='i-abcd1234',
message=u"♬ †øU \n†øU ¥ºu|m: T0µ ♪")
t.assert_equal(
u'_sc|my_check.name|{0}|d:{1}|h:i-abcd1234|#bar:baz,foo|m:{2}'.
format(self.statsd.WARNING, now, u"♬ †øU \\n†øU ¥ºu|m\: T0µ ♪"),
self.recv())
self.statsd.service_check('my_check.name',
self.statsd.WARNING,
tags=['key1:val1', 'key2:val2'],
timestamp=now,
hostname='i-abcd1234',
message=u"♬ †øU \n†øU ¥ºu|m: T0µ ♪")
t.assert_equal(
u'_sc|my_check.name|{0}|d:{1}|h:i-abcd1234|#key1:val1,key2:val2,bar:baz,foo|m:{2}'
.format(self.statsd.WARNING, now, u"♬ †øU \\n†øU ¥ºu|m\: T0µ ♪"),
self.recv())
def test_metric_namespace(self):
"""
Namespace prefixes all metric names.
"""
self.statsd.namespace = "foo"
self.statsd.gauge('gauge', 123.4)
t.assert_equal('foo.gauge:123.4|g', self.recv())
# Test Client level contant tags
def test_gauge_constant_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.gauge('gauge', 123.4)
assert self.recv() == 'gauge:123.4|g|#bar:baz,foo'
def test_counter_constant_tag_with_metric_level_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.increment('page.views', tags=['extra'])
t.assert_equal('page.views:1|c|#extra,bar:baz,foo', self.recv())
def test_gauge_constant_tags_with_metric_level_tags_twice(self):
metric_level_tag = ['foo:bar']
self.statsd.constant_tags = ['bar:baz']
self.statsd.gauge('gauge', 123.4, tags=metric_level_tag)
assert self.recv() == 'gauge:123.4|g|#foo:bar,bar:baz'
# sending metrics multiple times with same metric-level tags
# should not duplicate the tags being sent
self.statsd.gauge('gauge', 123.4, tags=metric_level_tag)
assert self.recv() == 'gauge:123.4|g|#foo:bar,bar:baz'
@staticmethod
def assert_almost_equal(a, b, delta):
assert 0 <= abs(a -
b) <= delta, "%s - %s not within %s" % (a, b, delta)
def test_socket_error(self):
self.statsd.socket = BrokenSocket()
self.statsd.gauge('no error', 1)
assert True, 'success'
def test_timed(self):
"""
Measure the distribution of a function's run time.
"""
# In seconds
@self.statsd.timed('timed.test')
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
t.assert_equal('func', func.__name__)
t.assert_equal('docstring', func.__doc__)
result = func(1, 2, d=3)
# Assert it handles args and kwargs correctly.
t.assert_equal(result, (1, 2, 1, 3))
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
# Repeat, force timer value in milliseconds
@self.statsd.timed('timed.test', use_ms=True)
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
func(1, 2, d=3)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(500, float(value), 100)
def test_timed_in_ms(self):
"""
Timed value is reported in ms when statsd.use_ms is True.
"""
# Arm statsd to use_ms
self.statsd.use_ms = True
# Sample a function run time
@self.statsd.timed('timed.test')
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
func(1, 2, d=3)
# Assess the packet
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(500, float(value), 100)
# Repeat, force timer value in seconds
@self.statsd.timed('timed.test', use_ms=False)
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
func(1, 2, d=3)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_no_metric(self, ):
"""
Test using a decorator without providing a metric.
"""
@self.statsd.timed()
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
t.assert_equal('func', func.__name__)
t.assert_equal('docstring', func.__doc__)
result = func(1, 2, d=3)
# Assert it handles args and kwargs correctly.
t.assert_equal(result, (1, 2, 1, 3))
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('tests.unit.dogstatsd.test_statsd.func', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_coroutine(self):
"""
Measure the distribution of a coroutine function's run time.
Warning: Python > 3.5 only.
"""
if not is_higher_py35():
raise SkipTest(
u"Coroutines are supported on Python 3.5 or higher.")
import asyncio
@self.statsd.timed('timed.test')
@asyncio.coroutine
def print_foo():
"""docstring"""
time.sleep(0.5)
print("foo")
loop = asyncio.get_event_loop()
loop.run_until_complete(print_foo())
loop.close()
# Assert
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_context(self):
"""
Measure the distribution of a context's run time.
"""
# In seconds
with self.statsd.timed('timed_context.test') as timer:
t.assert_is_instance(timer, TimedContextManagerDecorator)
time.sleep(0.5)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
self.assert_almost_equal(0.5, timer.elapsed, 0.1)
# In milliseconds
with self.statsd.timed('timed_context.test', use_ms=True) as timer:
time.sleep(0.5)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(500, float(value), 100)
self.assert_almost_equal(500, timer.elapsed, 100)
def test_timed_context_exception(self):
"""
Exception bubbles out of the `timed` context manager.
"""
class ContextException(Exception):
pass
def func(self):
with self.statsd.timed('timed_context.test.exception'):
time.sleep(0.5)
raise ContextException()
# Ensure the exception was raised.
t.assert_raises(ContextException, func, self)
# Ensure the timing was recorded.
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test.exception', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_context_no_metric_exception(self):
"""Test that an exception occurs if using a context manager without a metric."""
def func(self):
with self.statsd.timed():
time.sleep(0.5)
# Ensure the exception was raised.
t.assert_raises(TypeError, func, self)
# Ensure the timing was recorded.
packet = self.recv()
t.assert_equal(packet, None)
def test_timed_start_stop_calls(self):
# In seconds
timer = self.statsd.timed('timed_context.test')
timer.start()
time.sleep(0.5)
timer.stop()
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
# In milliseconds
timer = self.statsd.timed('timed_context.test', use_ms=True)
timer.start()
time.sleep(0.5)
timer.stop()
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(500, float(value), 100)
def test_batched(self):
self.statsd.open_buffer()
self.statsd.gauge('page.views', 123)
self.statsd.timing('timer', 123)
self.statsd.close_buffer()
t.assert_equal('page.views:123|g\ntimer:123|ms', self.recv())
def test_context_manager(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
statsd.gauge('page.views', 123)
statsd.timing('timer', 123)
t.assert_equal('page.views:123|g\ntimer:123|ms', fake_socket.recv())
def test_batched_buffer_autoflush(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
for i in range(51):
statsd.increment('mycounter')
t.assert_equal('\n'.join(['mycounter:1|c' for i in range(50)]),
fake_socket.recv())
t.assert_equal('mycounter:1|c', fake_socket.recv())
def test_module_level_instance(self):
t.assert_true(isinstance(statsd, DogStatsd))
def test_instantiating_does_not_connect(self):
dogpound = DogStatsd()
t.assert_equal(None, dogpound.socket)
def test_accessing_socket_opens_socket(self):
dogpound = DogStatsd()
try:
t.assert_not_equal(None, dogpound.get_socket())
finally:
dogpound.socket.close()
def test_accessing_socket_multiple_times_returns_same_socket(self):
dogpound = DogStatsd()
fresh_socket = FakeSocket()
dogpound.socket = fresh_socket
t.assert_equal(fresh_socket, dogpound.get_socket())
t.assert_not_equal(FakeSocket(), dogpound.get_socket())
def test_tags_from_environment(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
statsd = DogStatsd()
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal('gt:123.4|g|#country:china,age:45,blue',
statsd.socket.recv())
def test_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
statsd = DogStatsd(constant_tags=['country:canada', 'red'])
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal(
'gt:123.4|g|#country:canada,red,country:china,age:45,blue',
statsd.socket.recv())
def test_entity_tag_from_environment(self):
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ['DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
statsd = DogStatsd()
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal(
'gt:123.4|g|#dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d',
statsd.socket.recv())
def test_entity_tag_from_environment_and_constant(self):
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ['DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
statsd = DogStatsd(constant_tags=['country:canada', 'red'])
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal(
'gt:123.4|g|#country:canada,red,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d',
statsd.socket.recv())
def test_entity_tag_and_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ[
'DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
statsd = DogStatsd(constant_tags=['country:canada', 'red'])
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal(
'gt:123.4|g|#country:canada,red,country:china,age:45,blue,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d',
statsd.socket.recv())
def test_gauge_doesnt_send_None(self):
self.statsd.gauge('metric', None)
assert self.recv() is None
def test_increment_doesnt_send_None(self):
self.statsd.increment('metric', None)
assert self.recv() is None
def test_decrement_doesnt_send_None(self):
self.statsd.decrement('metric', None)
assert self.recv() is None
def test_timing_doesnt_send_None(self):
self.statsd.timing('metric', None)
assert self.recv() is None
def test_histogram_doesnt_send_None(self):
self.statsd.histogram('metric', None)
assert self.recv() is None
class TestDogStatsd(unittest.TestCase):
def setUp(self):
"""
Set up a default Dogstatsd instance and mock the proc filesystem.
"""
#
self.statsd = DogStatsd(telemetry_min_flush_interval=0)
self.statsd.socket = FakeSocket()
self.statsd._reset_telemetry()
# Mock the proc filesystem
route_data = load_fixtures('route')
self._procfs_mock = patch('datadog.util.compat.builtins.open',
mock_open())
self._procfs_mock.start(
).return_value.readlines.return_value = route_data.split("\n")
def tearDown(self):
"""
Unmock the proc filesystem.
"""
self._procfs_mock.stop()
def assert_equal_telemetry(self,
expected_payload,
actual_payload,
telemetry=None):
if telemetry is None:
telemetry = telemetry_metrics(bytes_sent=len(expected_payload))
if expected_payload:
expected_payload = "\n".join([expected_payload, telemetry])
else:
expected_payload = telemetry
return self.assertEqual(expected_payload, actual_payload)
def assert_almost_equal(self, val1, val2, delta):
return self.assertTrue(0 <= abs(val1 - val2) <= delta,
"%s - %s not within %s" % (val1, val2, delta))
def recv(self, count=1):
packets = []
for _ in range(count):
packets.append(self.statsd.socket.recv())
return "\n".join(packets)
def test_initialization(self):
"""
`initialize` overrides `statsd` default instance attributes.
"""
options = {'statsd_host': "myhost", 'statsd_port': 1234}
# Default values
self.assertEqual(statsd.host, "localhost")
self.assertEqual(statsd.port, 8125)
# After initialization
initialize(**options)
self.assertEqual(statsd.host, "myhost")
self.assertEqual(statsd.port, 1234)
# Add namespace
options['statsd_namespace'] = "mynamespace"
initialize(**options)
self.assertEqual(statsd.host, "myhost")
self.assertEqual(statsd.port, 1234)
self.assertEqual(statsd.namespace, "mynamespace")
# Set `statsd` host to the system's default route
initialize(statsd_use_default_route=True, **options)
self.assertEqual(statsd.host, "172.17.0.1")
self.assertEqual(statsd.port, 1234)
# Add UNIX socket
options['statsd_socket_path'] = '/var/run/dogstatsd.sock'
initialize(**options)
self.assertEqual(statsd.socket_path, options['statsd_socket_path'])
self.assertIsNone(statsd.host)
self.assertIsNone(statsd.port)
def test_dogstatsd_initialization_with_env_vars(self):
"""
Dogstatsd can retrieve its config from env vars when
not provided in constructor.
"""
# Setup
with preserve_environment_variable('DD_AGENT_HOST'):
os.environ['DD_AGENT_HOST'] = 'myenvvarhost'
with preserve_environment_variable('DD_DOGSTATSD_PORT'):
os.environ['DD_DOGSTATSD_PORT'] = '4321'
dogstatsd = DogStatsd()
# Assert
self.assertEqual(dogstatsd.host, "myenvvarhost")
self.assertEqual(dogstatsd.port, 4321)
def test_default_route(self):
"""
Dogstatsd host can be dynamically set to the default route.
"""
self.assertEqual(DogStatsd(use_default_route=True).host, "172.17.0.1")
def test_set(self):
self.statsd.set('set', 123)
self.assert_equal_telemetry('set:123|s', self.recv(2))
def test_gauge(self):
self.statsd.gauge('gauge', 123.4)
self.assert_equal_telemetry('gauge:123.4|g', self.recv(2))
def test_counter(self):
self.statsd.increment('page.views')
self.assert_equal_telemetry('page.views:1|c', self.recv(2))
self.statsd._reset_telemetry()
self.statsd.increment('page.views', 11)
self.assert_equal_telemetry('page.views:11|c', self.recv(2))
self.statsd._reset_telemetry()
self.statsd.decrement('page.views')
self.assert_equal_telemetry('page.views:-1|c', self.recv(2))
self.statsd._reset_telemetry()
self.statsd.decrement('page.views', 12)
self.assert_equal_telemetry('page.views:-12|c', self.recv(2))
def test_histogram(self):
self.statsd.histogram('histo', 123.4)
self.assert_equal_telemetry('histo:123.4|h', self.recv(2))
def test_pipe_in_tags(self):
self.statsd.gauge('gt', 123.4, tags=['pipe|in:tag', 'red'])
self.assert_equal_telemetry('gt:123.4|g|#pipe_in:tag,red',
self.recv(2))
def test_tagged_gauge(self):
self.statsd.gauge('gt',
123.4,
tags=['country:china', 'age:45', 'blue'])
self.assert_equal_telemetry('gt:123.4|g|#country:china,age:45,blue',
self.recv(2))
def test_tagged_counter(self):
self.statsd.increment('ct', tags=[u'country:españa', 'red'])
self.assert_equal_telemetry(u'ct:1|c|#country:españa,red',
self.recv(2))
def test_tagged_histogram(self):
self.statsd.histogram('h', 1, tags=['red'])
self.assert_equal_telemetry('h:1|h|#red', self.recv(2))
def test_sample_rate(self):
self.statsd._telemetry = False # disabling telemetry since sample_rate imply randomness
self.statsd.increment('c', sample_rate=0)
self.assertFalse(self.statsd.socket.recv())
for _ in range(10000):
self.statsd.increment('sampled_counter', sample_rate=0.3)
self.assert_almost_equal(3000, len(self.statsd.socket.payloads), 150)
self.assertEqual('sampled_counter:1|c|@0.3', self.recv())
def test_default_sample_rate(self):
self.statsd._telemetry = False # disabling telemetry since sample_rate imply randomness
self.statsd.default_sample_rate = 0.3
for _ in range(10000):
self.statsd.increment('sampled_counter')
self.assert_almost_equal(3000, len(self.statsd.socket.payloads), 150)
self.assertEqual('sampled_counter:1|c|@0.3', self.recv())
def test_tags_and_samples(self):
self.statsd._telemetry = False # disabling telemetry since sample_rate imply randomness
for _ in range(100):
self.statsd.gauge('gst', 23, tags=["sampled"], sample_rate=0.9)
self.assertEqual('gst:23|g|@0.9|#sampled', self.recv())
def test_timing(self):
self.statsd.timing('t', 123)
self.assert_equal_telemetry('t:123|ms', self.recv(2))
def test_event(self):
self.statsd.event('Title',
u'L1\nL2',
priority='low',
date_happened=1375296969)
event = u'_e{5,6}:Title|L1\\nL2|d:1375296969|p:low'
self.assert_equal_telemetry(event,
self.recv(2),
telemetry=telemetry_metrics(
metrics=0,
events=1,
bytes_sent=len(event)))
self.statsd._reset_telemetry()
self.statsd.event('Title',
u'♬ †øU †øU ¥ºu T0µ ♪',
aggregation_key='key',
tags=['t1', 't2:v2'])
event = u'_e{5,19}:Title|♬ †øU †øU ¥ºu T0µ ♪|k:key|#t1,t2:v2'
self.assert_equal_telemetry(event,
self.recv(2),
telemetry=telemetry_metrics(
metrics=0,
events=1,
bytes_sent=len(event)))
def test_event_constant_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.event('Title',
u'L1\nL2',
priority='low',
date_happened=1375296969)
event = u'_e{5,6}:Title|L1\\nL2|d:1375296969|p:low|#bar:baz,foo'
self.assert_equal_telemetry(event,
self.recv(2),
telemetry=telemetry_metrics(
metrics=0,
events=1,
tags="bar:baz,foo",
bytes_sent=len(event)))
self.statsd._reset_telemetry()
self.statsd.event('Title',
u'♬ †øU †øU ¥ºu T0µ ♪',
aggregation_key='key',
tags=['t1', 't2:v2'])
event = u'_e{5,19}:Title|♬ †øU †øU ¥ºu T0µ ♪|k:key|#t1,t2:v2,bar:baz,foo'
self.assert_equal_telemetry(event,
self.recv(2),
telemetry=telemetry_metrics(
metrics=0,
events=1,
tags="bar:baz,foo",
bytes_sent=len(event)))
def test_service_check(self):
now = int(time.time())
self.statsd.service_check('my_check.name',
self.statsd.WARNING,
tags=['key1:val1', 'key2:val2'],
timestamp=now,
hostname='i-abcd1234',
message=u"♬ †øU \n†øU ¥ºu|m: T0µ ♪")
check = u'_sc|my_check.name|{0}|d:{1}|h:i-abcd1234|#key1:val1,key2:val2|m:{2}'.format(
self.statsd.WARNING, now, u"♬ †øU \\n†øU ¥ºu|m\\: T0µ ♪")
self.assert_equal_telemetry(check,
self.recv(2),
telemetry=telemetry_metrics(
metrics=0,
service_checks=1,
bytes_sent=len(check)))
def test_service_check_constant_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
now = int(time.time())
self.statsd.service_check('my_check.name',
self.statsd.WARNING,
timestamp=now,
hostname='i-abcd1234',
message=u"♬ †øU \n†øU ¥ºu|m: T0µ ♪")
check = u'_sc|my_check.name|{0}|d:{1}|h:i-abcd1234|#bar:baz,foo|m:{2}'.format(
self.statsd.WARNING, now, u"♬ †øU \\n†øU ¥ºu|m\\: T0µ ♪")
self.assert_equal_telemetry(check,
self.recv(2),
telemetry=telemetry_metrics(
metrics=0,
service_checks=1,
tags="bar:baz,foo",
bytes_sent=len(check)))
self.statsd._reset_telemetry()
self.statsd.service_check('my_check.name',
self.statsd.WARNING,
tags=['key1:val1', 'key2:val2'],
timestamp=now,
hostname='i-abcd1234',
message=u"♬ †øU \n†øU ¥ºu|m: T0µ ♪")
check = u'_sc|my_check.name|{0}|d:{1}|h:i-abcd1234|#key1:val1,key2:val2,bar:baz,foo|m:{2}'.format(
self.statsd.WARNING, now, u"♬ †øU \\n†øU ¥ºu|m\\: T0µ ♪")
self.assert_equal_telemetry(check,
self.recv(2),
telemetry=telemetry_metrics(
metrics=0,
service_checks=1,
tags="bar:baz,foo",
bytes_sent=len(check)))
def test_metric_namespace(self):
"""
Namespace prefixes all metric names.
"""
self.statsd.namespace = "foo"
self.statsd.gauge('gauge', 123.4)
self.assert_equal_telemetry('foo.gauge:123.4|g', self.recv(2))
# Test Client level contant tags
def test_gauge_constant_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.gauge('gauge', 123.4)
metric = 'gauge:123.4|g|#bar:baz,foo'
self.assert_equal_telemetry(metric,
self.recv(2),
telemetry=telemetry_metrics(
tags="bar:baz,foo",
bytes_sent=len(metric)))
def test_counter_constant_tag_with_metric_level_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.increment('page.views', tags=['extra'])
metric = 'page.views:1|c|#extra,bar:baz,foo'
self.assert_equal_telemetry(metric,
self.recv(2),
telemetry=telemetry_metrics(
tags="bar:baz,foo",
bytes_sent=len(metric)))
def test_gauge_constant_tags_with_metric_level_tags_twice(self):
metric_level_tag = ['foo:bar']
self.statsd.constant_tags = ['bar:baz']
self.statsd.gauge('gauge', 123.4, tags=metric_level_tag)
metric = 'gauge:123.4|g|#foo:bar,bar:baz'
self.assert_equal_telemetry(
metric,
self.recv(2),
telemetry=telemetry_metrics(tags="bar:baz",
bytes_sent=len(metric)))
self.statsd._reset_telemetry()
# sending metrics multiple times with same metric-level tags
# should not duplicate the tags being sent
self.statsd.gauge('gauge', 123.4, tags=metric_level_tag)
metric = "gauge:123.4|g|#foo:bar,bar:baz"
self.assert_equal_telemetry(
metric,
self.recv(2),
telemetry=telemetry_metrics(tags="bar:baz",
bytes_sent=len(metric)))
def test_socket_error(self):
self.statsd.socket = BrokenSocket()
with mock.patch("datadog.dogstatsd.base.log") as mock_log:
self.statsd.gauge('no error', 1)
mock_log.error.assert_not_called()
mock_log.warning.assert_called_once_with(
"Error submitting packet: %s, dropping the packet and closing the socket",
mock.ANY,
)
def test_socket_overflown(self):
self.statsd.socket = OverflownSocket()
with mock.patch("datadog.dogstatsd.base.log") as mock_log:
self.statsd.gauge('no error', 1)
mock_log.error.assert_not_called()
calls = [
call("Socket send would block: %s, dropping the packet",
mock.ANY)
]
mock_log.debug.assert_has_calls(calls * 2)
def test_distributed(self):
"""
Measure the distribution of a function's run time using distribution custom metric.
"""
# In seconds
@self.statsd.distributed('distributed.test')
def func(arg1, arg2, kwarg1=1, kwarg2=1):
"""docstring"""
time.sleep(0.5)
return (arg1, arg2, kwarg1, kwarg2)
self.assertEqual('func', func.__name__)
self.assertEqual('docstring', func.__doc__)
result = func(1, 2, kwarg2=3)
# Assert it handles args and kwargs correctly.
self.assertEqual(result, (1, 2, 1, 3))
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('d', type_)
self.assertEqual('distributed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
# Repeat, force timer value in milliseconds
@self.statsd.distributed('distributed.test', use_ms=True)
def func(arg1, arg2, kwarg1=1, kwarg2=1):
"""docstring"""
time.sleep(0.5)
return (arg1, arg2, kwarg1, kwarg2)
func(1, 2, kwarg2=3)
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('d', type_)
self.assertEqual('distributed.test', name)
self.assert_almost_equal(500, float(value), 100)
def test_timed(self):
"""
Measure the distribution of a function's run time.
"""
# In seconds
@self.statsd.timed('timed.test')
def func(arg1, arg2, kwarg1=1, kwarg2=1):
"""docstring"""
time.sleep(0.5)
return (arg1, arg2, kwarg1, kwarg2)
self.assertEqual('func', func.__name__)
self.assertEqual('docstring', func.__doc__)
result = func(1, 2, kwarg2=3)
# Assert it handles args and kwargs correctly.
self.assertEqual(result, (1, 2, 1, 3))
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
# Repeat, force timer value in milliseconds
@self.statsd.timed('timed.test', use_ms=True)
def func(arg1, arg2, kwarg1=1, kwarg2=1):
"""docstring"""
time.sleep(0.5)
return (arg1, arg2, kwarg1, kwarg2)
func(1, 2, kwarg2=3)
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed.test', name)
self.assert_almost_equal(500, float(value), 100)
def test_timed_in_ms(self):
"""
Timed value is reported in ms when statsd.use_ms is True.
"""
# Arm statsd to use_ms
self.statsd.use_ms = True
# Sample a function run time
@self.statsd.timed('timed.test')
def func(arg1, arg2, kwarg1=1, kwarg2=1):
"""docstring"""
time.sleep(0.5)
return (arg1, arg2, kwarg1, kwarg2)
func(1, 2, kwarg2=3)
# Assess the packet
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed.test', name)
self.assert_almost_equal(500, float(value), 100)
# Repeat, force timer value in seconds
@self.statsd.timed('timed.test', use_ms=False)
def func(arg1, arg2, kwarg1=1, kwarg2=1):
"""docstring"""
time.sleep(0.5)
return (arg1, arg2, kwarg1, kwarg2)
func(1, 2, kwarg2=3)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_no_metric(self, ):
"""
Test using a decorator without providing a metric.
"""
@self.statsd.timed()
def func(arg1, arg2, kwarg1=1, kwarg2=1):
"""docstring"""
time.sleep(0.5)
return (arg1, arg2, kwarg1, kwarg2)
self.assertEqual('func', func.__name__)
self.assertEqual('docstring', func.__doc__)
result = func(1, 2, kwarg2=3)
# Assert it handles args and kwargs correctly.
self.assertEqual(result, (1, 2, 1, 3))
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('tests.unit.dogstatsd.test_statsd.func', name)
self.assert_almost_equal(0.5, float(value), 0.1)
@pytest.mark.skipif(
not is_higher_py35(),
reason="Coroutines are supported on Python 3.5 or higher.")
def test_timed_coroutine(self):
"""
Measure the distribution of a coroutine function's run time.
Warning: Python > 3.5 only.
"""
import asyncio
source = """
@self.statsd.timed('timed.test')
async def print_foo():
"docstring"
import time
time.sleep(0.5)
print("foo")
"""
exec(source, {}, locals())
loop = asyncio.get_event_loop()
loop.run_until_complete(locals()['print_foo']())
loop.close()
# Assert
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_context(self):
"""
Measure the distribution of a context's run time.
"""
# In seconds
with self.statsd.timed('timed_context.test') as timer:
self.assertTrue(isinstance(timer, TimedContextManagerDecorator))
time.sleep(0.5)
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed_context.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
self.assert_almost_equal(0.5, timer.elapsed, 0.1)
# In milliseconds
with self.statsd.timed('timed_context.test', use_ms=True) as timer:
time.sleep(0.5)
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed_context.test', name)
self.assert_almost_equal(500, float(value), 100)
self.assert_almost_equal(500, timer.elapsed, 100)
def test_timed_context_exception(self):
"""
Exception bubbles out of the `timed` context manager.
"""
class ContextException(Exception):
pass
def func(self):
with self.statsd.timed('timed_context.test.exception'):
time.sleep(0.5)
raise ContextException()
# Ensure the exception was raised.
with pytest.raises(ContextException):
func(self)
# Ensure the timing was recorded.
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed_context.test.exception', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_context_no_metric_exception(self):
"""Test that an exception occurs if using a context manager without a metric."""
def func(self):
with self.statsd.timed():
time.sleep(0.5)
# Ensure the exception was raised.
with pytest.raises(TypeError):
func(self)
# Ensure the timing was recorded.
packet = self.statsd.socket.recv()
self.assertIsNone(packet)
def test_timed_start_stop_calls(self):
# In seconds
timer = self.statsd.timed('timed_context.test')
timer.start()
time.sleep(0.5)
timer.stop()
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed_context.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
# In milliseconds
timer = self.statsd.timed('timed_context.test', use_ms=True)
timer.start()
time.sleep(0.5)
timer.stop()
packet = self.recv(2).split("\n")[0] # ignore telemetry packet
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
self.assertEqual('ms', type_)
self.assertEqual('timed_context.test', name)
self.assert_almost_equal(500, float(value), 100)
def test_batching(self):
self.statsd.open_buffer()
self.statsd.gauge('page.views', 123)
self.statsd.timing('timer', 123)
self.statsd.close_buffer()
expected = "page.views:123|g\ntimer:123|ms"
self.assert_equal_telemetry(expected,
self.recv(2),
telemetry=telemetry_metrics(
metrics=2, bytes_sent=len(expected)))
def test_batching_sequential(self):
self.statsd.open_buffer()
self.statsd.gauge('discarded.data', 123)
self.statsd.close_buffer()
self.statsd.open_buffer()
self.statsd.gauge('page.views', 123)
self.statsd.timing('timer', 123)
self.statsd.close_buffer()
expected1 = 'discarded.data:123|g'
expected_metrics1 = telemetry_metrics(metrics=1,
bytes_sent=len(expected1))
self.assert_equal_telemetry(expected1,
self.recv(2),
telemetry=expected_metrics1)
expected2 = "page.views:123|g\ntimer:123|ms"
self.assert_equal_telemetry(expected2,
self.recv(2),
telemetry=telemetry_metrics(
metrics=2,
packets_sent=2,
bytes_sent=len(expected2 +
expected_metrics1)))
def test_threaded_batching(self):
num_threads = 4
threads = []
def batch_metrics(index, dsd):
time.sleep(0.3 * index)
dsd.open_buffer()
time.sleep(0.1)
dsd.gauge('page.%d.views' % index, 123)
time.sleep(0.1)
dsd.timing('timer.%d' % index, 123)
time.sleep(0.5)
dsd.close_buffer()
for idx in range(num_threads):
threads.append(
Thread(target=batch_metrics, args=(idx, self.statsd)))
for thread in threads:
thread.start()
for thread in threads:
if thread.is_alive():
thread.join()
# This is a bit of a tricky thing to test for - initially only our data packet is
# sent but then telemetry is flushed/reset and the subsequent metric xmit includes
# the telemetry data for the previous packet. The reason for 726 -> 727 increase is
# because packet #2 sends a three digit byte count ("726") that then increases the
# next metric size by 1 byte.
expected_xfer_metrics = [
(33, 1),
(726, 2),
(727, 2),
(727, 2),
]
for idx in range(num_threads):
expected_message = "page.%d.views:123|g\ntimer.%d:123|ms" % (idx,
idx)
bytes_sent, packets_sent = expected_xfer_metrics[idx]
self.assert_equal_telemetry(expected_message,
self.recv(2),
telemetry=telemetry_metrics(
metrics=2,
bytes_sent=bytes_sent,
packets_sent=packets_sent,
))
def test_close_buffer_without_open(self):
dogstatsd = DogStatsd()
with self.assertRaises(BufferError):
dogstatsd.close_buffer()
def test_threaded_close_buffer_without_open(self):
def batch_metrics(dsd):
time.sleep(0.3)
dsd.open_buffer()
dsd.gauge('page.views', 123)
dsd.timing('timer', 123)
time.sleep(0.5)
dsd.close_buffer()
def close_async_buffer(self, dsd):
# Ensures that buffer is defined
dsd.open_buffer()
dsd.close_buffer()
time.sleep(0.5)
with self.assertRaises(RuntimeError):
dsd.close_buffer()
thread1 = Thread(target=batch_metrics, args=(self.statsd, ))
thread2 = Thread(target=close_async_buffer, args=(
self,
self.statsd,
))
for thread in [thread1, thread2]:
thread.start()
for thread in [thread1, thread2]:
if thread.is_alive():
thread.join()
expected_message = "page.views:123|g\ntimer:123|ms"
self.assert_equal_telemetry(expected_message,
self.recv(2),
telemetry=telemetry_metrics(
metrics=2,
bytes_sent=29,
packets_sent=1,
))
def test_telemetry(self):
self.statsd.metrics_count = 1
self.statsd.events_count = 2
self.statsd.service_checks_count = 3
self.statsd.bytes_sent = 4
self.statsd.bytes_dropped = 5
self.statsd.packets_sent = 6
self.statsd.packets_dropped = 7
self.statsd.open_buffer()
self.statsd.gauge('page.views', 123)
self.statsd.close_buffer()
payload = "page.views:123|g"
telemetry = telemetry_metrics(metrics=2,
events=2,
service_checks=3,
bytes_sent=4 + len(payload),
bytes_dropped=5,
packets_sent=7,
packets_dropped=7)
self.assert_equal_telemetry(payload, self.recv(2), telemetry=telemetry)
self.assertEqual(0, self.statsd.metrics_count)
self.assertEqual(0, self.statsd.events_count)
self.assertEqual(0, self.statsd.service_checks_count)
self.assertEqual(len(telemetry), self.statsd.bytes_sent)
self.assertEqual(0, self.statsd.bytes_dropped)
self.assertEqual(1, self.statsd.packets_sent)
self.assertEqual(0, self.statsd.packets_dropped)
def test_telemetry_flush_interval(self):
dogstatsd = DogStatsd()
fake_socket = FakeSocket()
dogstatsd.socket = fake_socket
# set the last flush time in the future to be sure we won't flush
dogstatsd._last_flush_time = time.time(
) + dogstatsd._telemetry_flush_interval
dogstatsd.gauge('gauge', 123.4)
metric = 'gauge:123.4|g'
self.assertEqual(metric, fake_socket.recv())
time1 = time.time()
# setting the last flush time in the past to trigger a telemetry flush
dogstatsd._last_flush_time = time1 - dogstatsd._telemetry_flush_interval - 1
dogstatsd.gauge('gauge', 123.4)
self.assert_equal_telemetry(metric,
fake_socket.recv(2),
telemetry=telemetry_metrics(
metrics=2,
bytes_sent=2 * len(metric),
packets_sent=2))
# assert that _last_flush_time has been updated
self.assertTrue(time1 < dogstatsd._last_flush_time)
def test_telemetry_flush_interval_alternate_destination(self):
dogstatsd = DogStatsd(telemetry_host='foo')
fake_socket = FakeSocket()
dogstatsd.socket = fake_socket
fake_telemetry_socket = FakeSocket()
dogstatsd.telemetry_socket = fake_telemetry_socket
self.assertIsNotNone(dogstatsd.telemetry_host)
self.assertIsNotNone(dogstatsd.telemetry_port)
self.assertTrue(dogstatsd._dedicated_telemetry_destination())
# set the last flush time in the future to be sure we won't flush
dogstatsd._last_flush_time = time.time(
) + dogstatsd._telemetry_flush_interval
dogstatsd.gauge('gauge', 123.4)
self.assertEqual('gauge:123.4|g', fake_socket.recv())
time1 = time.time()
# setting the last flush time in the past to trigger a telemetry flush
dogstatsd._last_flush_time = time1 - dogstatsd._telemetry_flush_interval - 1
dogstatsd.gauge('gauge', 123.4)
self.assertEqual('gauge:123.4|g', fake_socket.recv())
self.assert_equal_telemetry('',
fake_telemetry_socket.recv(),
telemetry=telemetry_metrics(
metrics=2,
bytes_sent=13 * 2,
packets_sent=2))
# assert that _last_flush_time has been updated
self.assertTrue(time1 < dogstatsd._last_flush_time)
def test_telemetry_flush_interval_batch(self):
dogstatsd = DogStatsd()
fake_socket = FakeSocket()
dogstatsd.socket = fake_socket
dogstatsd.open_buffer()
dogstatsd.gauge('gauge1', 1)
dogstatsd.gauge('gauge2', 2)
time1 = time.time()
# setting the last flush time in the past to trigger a telemetry flush
dogstatsd._last_flush_time = time1 - statsd._telemetry_flush_interval - 1
dogstatsd.close_buffer()
metric = 'gauge1:1|g\ngauge2:2|g'
self.assert_equal_telemetry(metric,
fake_socket.recv(2),
telemetry=telemetry_metrics(
metrics=2, bytes_sent=len(metric)))
# assert that _last_flush_time has been updated
self.assertTrue(time1 < dogstatsd._last_flush_time)
def test_context_manager(self):
fake_socket = FakeSocket()
with DogStatsd(telemetry_min_flush_interval=0) as dogstatsd:
dogstatsd.socket = fake_socket
dogstatsd.gauge('page.views', 123)
dogstatsd.timing('timer', 123)
metric = "page.views:123|g\ntimer:123|ms"
self.assertEqual(metric, fake_socket.recv())
self.assertEqual(telemetry_metrics(metrics=2, bytes_sent=len(metric)),
fake_socket.recv())
# self.assert_equal_telemetry("page.views:123|g\ntimer:123|ms", fake_socket.recv(2), telemetry=telemetry_metrics(metrics=2))
def test_batched_buffer_autoflush(self):
fake_socket = FakeSocket()
bytes_sent = 0
with DogStatsd(telemetry_min_flush_interval=0) as dogstatsd:
single_metric = 'mycounter:1|c'
self.assertEqual(dogstatsd._max_payload_size,
UDP_OPTIMAL_PAYLOAD_LENGTH)
metrics_per_packet = dogstatsd._max_payload_size // (
len(single_metric) + 1)
dogstatsd.socket = fake_socket
for _ in range(metrics_per_packet + 1):
dogstatsd.increment('mycounter')
payload = '\n'.join(
[single_metric for _ in range(metrics_per_packet)])
telemetry = telemetry_metrics(metrics=metrics_per_packet + 1,
bytes_sent=len(payload))
bytes_sent += len(payload) + len(telemetry)
self.assertEqual(payload, fake_socket.recv())
self.assertEqual(telemetry, fake_socket.recv())
self.assertEqual(single_metric, fake_socket.recv())
telemetry = telemetry_metrics(metrics=0,
packets_sent=2,
bytes_sent=len(single_metric) +
len(telemetry))
self.assertEqual(telemetry, fake_socket.recv())
def test_module_level_instance(self):
self.assertTrue(isinstance(statsd, DogStatsd))
def test_instantiating_does_not_connect(self):
dogpound = DogStatsd()
self.assertIsNone(dogpound.socket)
def test_accessing_socket_opens_socket(self):
dogpound = DogStatsd()
try:
self.assertIsNotNone(dogpound.get_socket())
finally:
dogpound.socket.close()
def test_accessing_socket_multiple_times_returns_same_socket(self):
dogpound = DogStatsd()
fresh_socket = FakeSocket()
dogpound.socket = fresh_socket
self.assertEqual(fresh_socket, dogpound.get_socket())
self.assertNotEqual(FakeSocket(), dogpound.get_socket())
def test_tags_from_environment(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
dogstatsd = DogStatsd(telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
dogstatsd.gauge('gt', 123.4)
metric = 'gt:123.4|g|#country:china,age:45,blue'
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(
telemetry_metrics(tags="country:china,age:45,blue",
bytes_sent=len(metric)), dogstatsd.socket.recv())
def test_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
dogstatsd = DogStatsd(constant_tags=['country:canada', 'red'],
telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
dogstatsd.gauge('gt', 123.4)
tags = "country:canada,red,country:china,age:45,blue"
metric = 'gt:123.4|g|#' + tags
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(telemetry_metrics(tags=tags, bytes_sent=len(metric)),
dogstatsd.socket.recv())
def test_entity_tag_from_environment(self):
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ['DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
dogstatsd = DogStatsd(telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
dogstatsd.gauge('gt', 123.4)
metric = 'gt:123.4|g|#dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d'
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(
telemetry_metrics(
tags=
"dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d",
bytes_sent=len(metric)), dogstatsd.socket.recv())
def test_entity_tag_from_environment_and_constant(self):
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ['DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
dogstatsd = DogStatsd(constant_tags=['country:canada', 'red'],
telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
dogstatsd.gauge('gt', 123.4)
metric = 'gt:123.4|g|#country:canada,red,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d'
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(
telemetry_metrics(
tags=
"country:canada,red,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d",
bytes_sent=len(metric)), dogstatsd.socket.recv())
def test_entity_tag_and_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ[
'DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
dogstatsd = DogStatsd(constant_tags=['country:canada', 'red'],
telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
dogstatsd.gauge('gt', 123.4)
tags = "country:canada,red,country:china,age:45,blue,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d"
metric = 'gt:123.4|g|#' + tags
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(telemetry_metrics(tags=tags, bytes_sent=len(metric)),
dogstatsd.socket.recv())
def test_dogstatsd_initialization_with_dd_env_service_version(self):
"""
Dogstatsd should automatically use DD_ENV, DD_SERVICE, and DD_VERSION (if present)
to set {env, service, version} as global tags for all metrics emitted.
"""
cases = [
# Test various permutations of setting DD_* env vars, as well as other global tag configuration.
# An empty string signifies that the env var either isn't set or that it is explicitly set to empty string.
('', '', '', '', [], []),
('prod', '', '', '', [], ['env:prod']),
('prod', 'dog', '', '', [], ['env:prod', 'service:dog']),
('prod', 'dog', 'abc123', '', [],
['env:prod', 'service:dog', 'version:abc123']),
('prod', 'dog', 'abc123', 'env:prod,type:app', [], [
'env:prod', 'env:prod', 'service:dog', 'type:app',
'version:abc123'
]),
('prod', 'dog', 'abc123', 'env:prod2,type:app', [], [
'env:prod', 'env:prod2', 'service:dog', 'type:app',
'version:abc123'
]),
('prod', 'dog', 'abc123', '', ['env:prod', 'type:app'], [
'env:prod', 'env:prod', 'service:dog', 'type:app',
'version:abc123'
]),
('prod', 'dog', 'abc123', '', ['env:prod2', 'type:app'], [
'env:prod', 'env:prod2', 'service:dog', 'type:app',
'version:abc123'
]),
('prod', 'dog', 'abc123', 'env:prod3,custom_tag:cat',
['env:prod2', 'type:app'], [
'custom_tag:cat', 'env:prod', 'env:prod2', 'env:prod3',
'service:dog', 'type:app', 'version:abc123'
]),
]
for case in cases:
dd_env, dd_service, dd_version, datadog_tags, constant_tags, global_tags = case
with EnvVars(
env_vars={
'DATADOG_TAGS': datadog_tags,
'DD_ENV': dd_env,
'DD_SERVICE': dd_service,
'DD_VERSION': dd_version,
}):
dogstatsd = DogStatsd(constant_tags=constant_tags,
telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
# Guarantee consistent ordering, regardless of insertion order.
dogstatsd.constant_tags.sort()
self.assertEqual(global_tags, dogstatsd.constant_tags)
# Make call with no tags passed; only the globally configured tags will be used.
global_tags_str = ','.join([t for t in global_tags])
dogstatsd.gauge('gt', 123.4)
# Protect against the no tags case.
metric = 'gt:123.4|g|#{}'.format(
global_tags_str) if global_tags_str else 'gt:123.4|g'
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(
telemetry_metrics(tags=global_tags_str,
bytes_sent=len(metric)),
dogstatsd.socket.recv())
dogstatsd._reset_telemetry()
# Make another call with local tags passed.
passed_tags = ['env:prod', 'version:def456', 'custom_tag:toad']
all_tags_str = ','.join([t for t in passed_tags + global_tags])
dogstatsd.gauge('gt', 123.4, tags=passed_tags)
metric = 'gt:123.4|g|#{}'.format(all_tags_str)
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(
telemetry_metrics(tags=global_tags_str,
bytes_sent=len(metric)),
dogstatsd.socket.recv())
def test_gauge_does_not_send_none(self):
self.statsd.gauge('metric', None)
self.assertIsNone(self.statsd.socket.recv())
def test_increment_does_not_send_none(self):
self.statsd.increment('metric', None)
self.assertIsNone(self.statsd.socket.recv())
def test_decrement_does_not_send_none(self):
self.statsd.decrement('metric', None)
self.assertIsNone(self.statsd.socket.recv())
def test_timing_does_not_send_none(self):
self.statsd.timing('metric', None)
self.assertIsNone(self.statsd.socket.recv())
def test_histogram_does_not_send_none(self):
self.statsd.histogram('metric', None)
self.assertIsNone(self.statsd.socket.recv())
class TestDogStatsd(object):
def setUp(self):
self.statsd = DogStatsd()
self.statsd.socket = FakeSocket()
def recv(self):
return self.statsd.socket.recv()
def test_initialization(self):
options = {'statsd_host': "myhost", 'statsd_port': 1234}
t.assert_equal(statsd.host, "localhost")
t.assert_equal(statsd.port, 8125)
initialize(**options)
t.assert_equal(statsd.host, "myhost")
t.assert_equal(statsd.port, 1234)
def test_default_route(self):
options = {
'statsd_use_default_route': True,
}
initialize(**options)
t.assert_equal(statsd.use_default_route, True)
def test_set(self):
self.statsd.set('set', 123)
assert self.recv() == 'set:123|s'
def test_gauge(self):
self.statsd.gauge('gauge', 123.4)
assert self.recv() == 'gauge:123.4|g'
def test_counter(self):
self.statsd.increment('page.views')
t.assert_equal('page.views:1|c', self.recv())
self.statsd.increment('page.views', 11)
t.assert_equal('page.views:11|c', self.recv())
self.statsd.decrement('page.views')
t.assert_equal('page.views:-1|c', self.recv())
self.statsd.decrement('page.views', 12)
t.assert_equal('page.views:-12|c', self.recv())
def test_histogram(self):
self.statsd.histogram('histo', 123.4)
t.assert_equal('histo:123.4|h', self.recv())
def test_tagged_gauge(self):
self.statsd.gauge('gt',
123.4,
tags=['country:china', 'age:45', 'blue'])
t.assert_equal('gt:123.4|g|#country:china,age:45,blue', self.recv())
def test_tagged_counter(self):
self.statsd.increment('ct', tags=[u'country:españa', 'red'])
t.assert_equal(u'ct:1|c|#country:españa,red', self.recv())
def test_tagged_histogram(self):
self.statsd.histogram('h', 1, tags=['red'])
t.assert_equal('h:1|h|#red', self.recv())
def test_sample_rate(self):
self.statsd.increment('c', sample_rate=0)
assert not self.recv()
for i in range(10000):
self.statsd.increment('sampled_counter', sample_rate=0.3)
self.assert_almost_equal(3000, len(self.statsd.socket.payloads), 150)
t.assert_equal('sampled_counter:1|c|@0.3', self.recv())
def test_tags_and_samples(self):
for i in range(100):
self.statsd.gauge('gst', 23, tags=["sampled"], sample_rate=0.9)
def test_tags_and_samples(self):
for i in range(100):
self.statsd.gauge('gst', 23, tags=["sampled"], sample_rate=0.9)
t.assert_equal('gst:23|g|@0.9|#sampled')
def test_timing(self):
self.statsd.timing('t', 123)
t.assert_equal('t:123|ms', self.recv())
def test_event(self):
self.statsd.event('Title',
u'L1\nL2',
priority='low',
date_happened=1375296969)
t.assert_equal(u'_e{5,6}:Title|L1\\nL2|d:1375296969|p:low',
self.recv())
self.statsd.event('Title',
u'♬ †øU †øU ¥ºu T0µ ♪',
aggregation_key='key',
tags=['t1', 't2:v2'])
t.assert_equal(u'_e{5,19}:Title|♬ †øU †øU ¥ºu T0µ ♪|k:key|#t1,t2:v2',
self.recv())
def test_event_constant_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.event('Title',
u'L1\nL2',
priority='low',
date_happened=1375296969)
t.assert_equal(
u'_e{5,6}:Title|L1\\nL2|d:1375296969|p:low|#bar:baz,foo',
self.recv())
self.statsd.event('Title',
u'♬ †øU †øU ¥ºu T0µ ♪',
aggregation_key='key',
tags=['t1', 't2:v2'])
t.assert_equal(
u'_e{5,19}:Title|♬ †øU †øU ¥ºu T0µ ♪|k:key|#t1,t2:v2,bar:baz,foo',
self.recv())
def test_service_check(self):
now = int(time.time())
self.statsd.service_check('my_check.name',
self.statsd.WARNING,
tags=['key1:val1', 'key2:val2'],
timestamp=now,
hostname='i-abcd1234',
message=u"♬ †øU \n†øU ¥ºu|m: T0µ ♪")
t.assert_equal(
u'_sc|my_check.name|{0}|d:{1}|h:i-abcd1234|#key1:val1,key2:val2|m:{2}'
.format(self.statsd.WARNING, now, u"♬ †øU \\n†øU ¥ºu|m\: T0µ ♪"),
self.recv())
def test_metric_namespace(self):
"""
Namespace prefixes all metric names.
"""
self.statsd.namespace = "foo"
self.statsd.gauge('gauge', 123.4)
t.assert_equal('foo.gauge:123.4|g', self.recv())
# Test Client level contant tags
def test_gauge_constant_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.gauge('gauge', 123.4)
assert self.recv() == 'gauge:123.4|g|#bar:baz,foo'
def test_counter_constant_tag_with_metric_level_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.increment('page.views', tags=['extra'])
t.assert_equal('page.views:1|c|#extra,bar:baz,foo', self.recv())
def test_gauge_constant_tags_with_metric_level_tags_twice(self):
metric_level_tag = ['foo:bar']
self.statsd.constant_tags = ['bar:baz']
self.statsd.gauge('gauge', 123.4, tags=metric_level_tag)
assert self.recv() == 'gauge:123.4|g|#foo:bar,bar:baz'
# sending metrics multiple times with same metric-level tags
# should not duplicate the tags being sent
self.statsd.gauge('gauge', 123.4, tags=metric_level_tag)
assert self.recv() == 'gauge:123.4|g|#foo:bar,bar:baz'
@staticmethod
def assert_almost_equal(a, b, delta):
assert 0 <= abs(a -
b) <= delta, "%s - %s not within %s" % (a, b, delta)
def test_socket_error(self):
self.statsd.socket = BrokenSocket()
self.statsd.gauge('no error', 1)
assert True, 'success'
def test_timed(self):
"""
Measure the distribution of a function's run time.
"""
# In seconds
@self.statsd.timed('timed.test')
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
t.assert_equal('func', func.__name__)
t.assert_equal('docstring', func.__doc__)
result = func(1, 2, d=3)
# Assert it handles args and kwargs correctly.
t.assert_equal(result, (1, 2, 1, 3))
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
# Repeat, force timer value in milliseconds
@self.statsd.timed('timed.test', use_ms=True)
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
func(1, 2, d=3)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(500, float(value), 100)
def test_timed_in_ms(self):
"""
Timed value is reported in ms when statsd.use_ms is True.
"""
# Arm statsd to use_ms
self.statsd.use_ms = True
# Sample a function run time
@self.statsd.timed('timed.test')
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
func(1, 2, d=3)
# Assess the packet
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(500, float(value), 100)
# Repeat, force timer value in seconds
@self.statsd.timed('timed.test', use_ms=False)
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
func(1, 2, d=3)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_no_metric(self, ):
"""
Test using a decorator without providing a metric.
"""
@self.statsd.timed()
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
t.assert_equal('func', func.__name__)
t.assert_equal('docstring', func.__doc__)
result = func(1, 2, d=3)
# Assert it handles args and kwargs correctly.
t.assert_equal(result, (1, 2, 1, 3))
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('tests.unit.dogstatsd.test_statsd.func', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_context(self):
"""
Measure the distribution of a context's run time.
"""
# In seconds
with self.statsd.timed('timed_context.test') as timer:
t.assert_is_instance(timer,
DogStatsd._TimedContextManagerDecorator)
time.sleep(0.5)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
self.assert_almost_equal(0.5, timer.elapsed, 0.1)
# In milliseconds
with self.statsd.timed('timed_context.test', use_ms=True) as timer:
time.sleep(0.5)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(500, float(value), 100)
self.assert_almost_equal(500, timer.elapsed, 100)
def test_timed_context_exception(self):
"""
Exception bubbles out of the `timed` context manager.
"""
class ContextException(Exception):
pass
def func(self):
with self.statsd.timed('timed_context.test.exception'):
time.sleep(0.5)
raise ContextException()
# Ensure the exception was raised.
t.assert_raises(ContextException, func, self)
# Ensure the timing was recorded.
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test.exception', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_context_no_metric_exception(self):
"""Test that an exception occurs if using a context manager without a metric."""
def func(self):
with self.statsd.timed():
time.sleep(0.5)
# Ensure the exception was raised.
t.assert_raises(TypeError, func, self)
# Ensure the timing was recorded.
packet = self.recv()
t.assert_equal(packet, None)
def test_timed_start_stop_calls(self):
# In seconds
timer = self.statsd.timed('timed_context.test')
timer.start()
time.sleep(0.5)
timer.stop()
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
# In milliseconds
timer = self.statsd.timed('timed_context.test', use_ms=True)
timer.start()
time.sleep(0.5)
timer.stop()
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(500, float(value), 100)
def test_batched(self):
self.statsd.open_buffer()
self.statsd.gauge('page.views', 123)
self.statsd.timing('timer', 123)
self.statsd.close_buffer()
t.assert_equal('page.views:123|g\ntimer:123|ms', self.recv())
def test_context_manager(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
statsd.gauge('page.views', 123)
statsd.timing('timer', 123)
t.assert_equal('page.views:123|g\ntimer:123|ms', fake_socket.recv())
def test_batched_buffer_autoflush(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
for i in range(51):
statsd.increment('mycounter')
t.assert_equal('\n'.join(['mycounter:1|c' for i in range(50)]),
fake_socket.recv())
t.assert_equal('mycounter:1|c', fake_socket.recv())
def test_module_level_instance(self):
t.assert_true(isinstance(statsd, DogStatsd))
def test_instantiating_does_not_connect(self):
dogpound = DogStatsd()
t.assert_equal(None, dogpound.socket)
def test_accessing_socket_opens_socket(self):
dogpound = DogStatsd()
try:
t.assert_not_equal(None, dogpound.get_socket())
finally:
dogpound.socket.close()
def test_accessing_socket_multiple_times_returns_same_socket(self):
dogpound = DogStatsd()
fresh_socket = FakeSocket()
dogpound.socket = fresh_socket
t.assert_equal(fresh_socket, dogpound.get_socket())
t.assert_not_equal(FakeSocket(), dogpound.get_socket())
def test_tags_from_environment(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
statsd = DogStatsd()
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal('gt:123.4|g|#country:china,age:45,blue',
statsd.socket.recv())
def test_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
statsd = DogStatsd(constant_tags=['country:canada', 'red'])
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal(
'gt:123.4|g|#country:canada,red,country:china,age:45,blue',
statsd.socket.recv())
def test_gauge_doesnt_send_None(self):
self.statsd.gauge('metric', None)
assert self.recv() is None
def test_increment_doesnt_send_None(self):
self.statsd.increment('metric', None)
assert self.recv() is None
def test_decrement_doesnt_send_None(self):
self.statsd.decrement('metric', None)
assert self.recv() is None
def test_timing_doesnt_send_None(self):
self.statsd.timing('metric', None)
assert self.recv() is None
def test_histogram_doesnt_send_None(self):
self.statsd.histogram('metric', None)
assert self.recv() is None
class StatsD(object):
def __init__(self, app, config, statsd=None):
self.config = config
for key, value in DEFAULTS.items():
self.config.setdefault(key, value)
self.statsd = DogStatsd(host=self.config['STATSD_HOST'],
port=self.config['STATSD_PORT'],
max_buffer_size=self.config['STATSD_MAX_BUFFER_SIZE'],
namespace=self.config['STATSD_NAMESPACE'],
constant_tags=self.config['STATSD_TAGS'],
use_ms=self.config['STATSD_USEMS']) \
if statsd is None \
else statsd
self.app = app
def timer(self, *args, **kwargs):
return TimerWrapper(self.statsd, *args, **kwargs)
def incr(self, *args, **kwargs):
return self.statsd.increment(*args, **kwargs)
def decr(self, *args, **kwargs):
return self.statsd.decrement(*args, **kwargs)
def initialize_lifecycle_hooks(self):
self.app.before_request(self.before_request)
self.app.after_request(self.after_request)
def before_request(self):
g.flask_datadog_start_time = time.time()
g.flask_datadog_request_tags = []
if self.config['DATADOG_RESPONSE_AUTO_TAG']:
self.add_request_tags([
'{tag_name}:{endpoint}'.format(
tag_name=self.config['DATADOG_RESPONSE_ENDPOINT_TAG_NAME'],
endpoint=str(request.endpoint).lower()),
'{tag_name}:{method}'.format(
tag_name=self.config['DATADOG_RESPONSE_METHOD_TAG_NAME'],
method=request.method.lower()),
])
def after_request(self, response):
if not hasattr(g, 'flask_datadog_start_time'):
return response
elapsed = time.time() - g.flask_datadog_start_time
if self.use_ms:
elapsed = int(round(1000 * elapsed))
if self.config['DATADOG_RESPONSE_AUTO_TAG']:
self.add_request_tags(
['status_code:%s' % (response.status_code, )])
tags = self.get_request_tags()
sample_rate = self.config['DATADOG_RESPONSE_SAMPLE_RATE']
self.statsd.timing(self.config['DATADOG_RESPONSE_METRIC_NAME'],
elapsed, tags, sample_rate)
if 'content-length' in response.headers:
size = int(response.headers['content-length'])
self.statsd.histogram(
self.config['DATADOG_RESPONSE_SIZE_METRIC_NAME'], size, tags,
sample_rate)
return response
def get_request_tags(self):
return getattr(g, 'flask_datadog_request_tags', [])
def add_request_tags(self, tags):
current_tags = self.get_request_tags()
g.flask_datadog_request_tags = current_tags + tags
return g.flask_datadog_request_tags
def __getattr__(self, name):
if self.statsd and hasattr(self.statsd, name):
return getattr(self.statsd, name)
raise AttributeError(
'\'StatsD\' has attribute \'{name}\''.format(name=name))
def __enter__(self):
return self.statsd.__enter__()
def __exit__(self, *args, **kwargs):
return self.statsd.__exit__(*args, **kwargs)
class TestDogStatsd(object):
def setUp(self):
self.statsd = DogStatsd()
self.statsd.socket = FakeSocket()
def recv(self):
return self.statsd.socket.recv()
def test_initialization(self):
options = {
'statsd_host': "myhost",
'statsd_port': 1234
}
t.assert_equal(statsd.host, "localhost")
t.assert_equal(statsd.port, 8125)
initialize(**options)
t.assert_equal(statsd.host, "myhost")
t.assert_equal(statsd.port, 1234)
def test_set(self):
self.statsd.set('set', 123)
assert self.recv() == 'set:123|s'
def test_gauge(self):
self.statsd.gauge('gauge', 123.4)
assert self.recv() == 'gauge:123.4|g'
def test_counter(self):
self.statsd.increment('page.views')
t.assert_equal('page.views:1|c', self.recv())
self.statsd.increment('page.views', 11)
t.assert_equal('page.views:11|c', self.recv())
self.statsd.decrement('page.views')
t.assert_equal('page.views:-1|c', self.recv())
self.statsd.decrement('page.views', 12)
t.assert_equal('page.views:-12|c', self.recv())
def test_histogram(self):
self.statsd.histogram('histo', 123.4)
t.assert_equal('histo:123.4|h', self.recv())
def test_tagged_gauge(self):
self.statsd.gauge('gt', 123.4, tags=['country:china', 'age:45', 'blue'])
t.assert_equal('gt:123.4|g|#country:china,age:45,blue', self.recv())
def test_tagged_counter(self):
self.statsd.increment('ct', tags=['country:canada', 'red'])
t.assert_equal('ct:1|c|#country:canada,red', self.recv())
def test_tagged_histogram(self):
self.statsd.histogram('h', 1, tags=['red'])
t.assert_equal('h:1|h|#red', self.recv())
def test_sample_rate(self):
self.statsd.increment('c', sample_rate=0)
assert not self.recv()
for i in range(10000):
self.statsd.increment('sampled_counter', sample_rate=0.3)
self.assert_almost_equal(3000, len(self.statsd.socket.payloads), 150)
t.assert_equal('sampled_counter:1|c|@0.3', self.recv())
def test_tags_and_samples(self):
for i in range(100):
self.statsd.gauge('gst', 23, tags=["sampled"], sample_rate=0.9)
def test_tags_and_samples(self):
for i in range(100):
self.statsd.gauge('gst', 23, tags=["sampled"], sample_rate=0.9)
t.assert_equal('gst:23|g|@0.9|#sampled')
def test_timing(self):
self.statsd.timing('t', 123)
t.assert_equal('t:123|ms', self.recv())
def test_event(self):
self.statsd.event('Title', u'L1\nL2', priority='low', date_happened=1375296969)
t.assert_equal(u'_e{5,6}:Title|L1\\nL2|d:1375296969|p:low', self.recv())
self.statsd.event('Title', u'♬ †øU †øU ¥ºu T0µ ♪',
aggregation_key='key', tags=['t1', 't2:v2'])
t.assert_equal(u'_e{5,19}:Title|♬ †øU †øU ¥ºu T0µ ♪|k:key|#t1,t2:v2', self.recv())
def test_event_constant_tags(self):
self.statsd.constant_tags = ['bar:baz', 'foo']
self.statsd.event('Title', u'L1\nL2', priority='low', date_happened=1375296969)
t.assert_equal(u'_e{5,6}:Title|L1\\nL2|d:1375296969|p:low|#bar:baz,foo', self.recv())
self.statsd.event('Title', u'♬ †øU †øU ¥ºu T0µ ♪',
aggregation_key='key', tags=['t1', 't2:v2'])
t.assert_equal(u'_e{5,19}:Title|♬ †øU †øU ¥ºu T0µ ♪|k:key|#t1,t2:v2,bar:baz,foo', self.recv())
def test_service_check(self):
now = int(time.time())
self.statsd.service_check(
'my_check.name', self.statsd.WARNING,
tags=['key1:val1', 'key2:val2'], timestamp=now,
hostname='i-abcd1234', message=u"♬ †øU \n†øU ¥ºu|m: T0µ ♪")
t.assert_equal(
u'_sc|my_check.name|{0}|d:{1}|h:i-abcd1234|#key1:val1,key2:val2|m:{2}'
.format(self.statsd.WARNING, now, u"♬ †øU \\n†øU ¥ºu|m\: T0µ ♪"), self.recv())
def test_metric_namespace(self):
"""
Namespace prefixes all metric names.
"""
self.statsd.namespace = "foo"
self.statsd.gauge('gauge', 123.4)
t.assert_equal('foo.gauge:123.4|g', self.recv())
# Test Client level contant tags
def test_gauge_constant_tags(self):
self.statsd.constant_tags=['bar:baz', 'foo']
self.statsd.gauge('gauge', 123.4)
assert self.recv() == 'gauge:123.4|g|#bar:baz,foo'
def test_counter_constant_tag_with_metric_level_tags(self):
self.statsd.constant_tags=['bar:baz', 'foo']
self.statsd.increment('page.views', tags=['extra'])
t.assert_equal('page.views:1|c|#extra,bar:baz,foo', self.recv())
def test_gauge_constant_tags_with_metric_level_tags_twice(self):
metric_level_tag = ['foo:bar']
self.statsd.constant_tags=['bar:baz']
self.statsd.gauge('gauge', 123.4, tags=metric_level_tag)
assert self.recv() == 'gauge:123.4|g|#foo:bar,bar:baz'
# sending metrics multiple times with same metric-level tags
# should not duplicate the tags being sent
self.statsd.gauge('gauge', 123.4, tags=metric_level_tag)
assert self.recv() == 'gauge:123.4|g|#foo:bar,bar:baz'
@staticmethod
def assert_almost_equal(a, b, delta):
assert 0 <= abs(a - b) <= delta, "%s - %s not within %s" % (a, b, delta)
def test_socket_error(self):
self.statsd.socket = BrokenSocket()
self.statsd.gauge('no error', 1)
assert True, 'success'
def test_timed(self):
"""
Measure the distribution of a function's run time.
"""
# In seconds
@self.statsd.timed('timed.test')
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
t.assert_equal('func', func.__name__)
t.assert_equal('docstring', func.__doc__)
result = func(1, 2, d=3)
# Assert it handles args and kwargs correctly.
t.assert_equal(result, (1, 2, 1, 3))
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
# Repeat, force timer value in milliseconds
@self.statsd.timed('timed.test', use_ms=True)
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
func(1, 2, d=3)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(500, float(value), 100)
def test_timed_in_ms(self):
"""
Timed value is reported in ms when statsd.use_ms is True.
"""
# Arm statsd to use_ms
self.statsd.use_ms = True
# Sample a function run time
@self.statsd.timed('timed.test')
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
func(1, 2, d=3)
# Assess the packet
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(500, float(value), 100)
# Repeat, force timer value in seconds
@self.statsd.timed('timed.test', use_ms=False)
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
func(1, 2, d=3)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_no_metric(self, ):
"""
Test using a decorator without providing a metric.
"""
@self.statsd.timed()
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
t.assert_equal('func', func.__name__)
t.assert_equal('docstring', func.__doc__)
result = func(1, 2, d=3)
# Assert it handles args and kwargs correctly.
t.assert_equal(result, (1, 2, 1, 3))
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('tests.unit.dogstatsd.test_statsd.func', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_context(self):
"""
Measure the distribution of a context's run time.
"""
# In seconds
with self.statsd.timed('timed_context.test'):
time.sleep(0.5)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
# In milliseconds
with self.statsd.timed('timed_context.test', use_ms=True):
time.sleep(0.5)
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test', name)
self.assert_almost_equal(500, float(value), 100)
def test_timed_context_exception(self):
"""
Exception bubbles out of the `timed` context manager.
"""
class ContextException(Exception):
pass
def func(self):
with self.statsd.timed('timed_context.test.exception'):
time.sleep(0.5)
raise ContextException()
# Ensure the exception was raised.
t.assert_raises(ContextException, func, self)
# Ensure the timing was recorded.
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed_context.test.exception', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_timed_context_no_metric_exception(self):
"""Test that an exception occurs if using a context manager without a metric."""
def func(self):
with self.statsd.timed():
time.sleep(0.5)
# Ensure the exception was raised.
t.assert_raises(TypeError, func, self)
# Ensure the timing was recorded.
packet = self.recv()
t.assert_equal(packet, None)
def test_batched(self):
self.statsd.open_buffer()
self.statsd.gauge('page.views', 123)
self.statsd.timing('timer', 123)
self.statsd.close_buffer()
t.assert_equal('page.views:123|g\ntimer:123|ms', self.recv())
def test_context_manager(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
statsd.gauge('page.views', 123)
statsd.timing('timer', 123)
t.assert_equal('page.views:123|g\ntimer:123|ms', fake_socket.recv())
def test_batched_buffer_autoflush(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
for i in range(51):
statsd.increment('mycounter')
t.assert_equal('\n'.join(['mycounter:1|c' for i in range(50)]), fake_socket.recv())
t.assert_equal('mycounter:1|c', fake_socket.recv())
def test_module_level_instance(self):
t.assert_true(isinstance(statsd, DogStatsd))
def test_instantiating_does_not_connect(self):
dogpound = DogStatsd()
t.assert_equal(None, dogpound.socket)
def test_accessing_socket_opens_socket(self):
dogpound = DogStatsd()
try:
t.assert_not_equal(None, dogpound.get_socket())
finally:
dogpound.socket.close()
def test_accessing_socket_multiple_times_returns_same_socket(self):
dogpound = DogStatsd()
fresh_socket = FakeSocket()
dogpound.socket = fresh_socket
t.assert_equal(fresh_socket, dogpound.get_socket())
t.assert_not_equal(FakeSocket(), dogpound.get_socket())
def test_tags_from_environment(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
statsd = DogStatsd()
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal('gt:123.4|g|#country:china,age:45,blue', statsd.socket.recv())
def test_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
statsd = DogStatsd(constant_tags=['country:canada', 'red'])
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal('gt:123.4|g|#country:canada,red,country:china,age:45,blue', statsd.socket.recv())
class TestDogStatsd(object):
def setUp(self):
self.statsd = DogStatsd()
self.statsd.socket = FakeSocket()
def recv(self):
return self.statsd.socket.recv()
def test_initialization(self):
options = {
'statsd_host': "myhost",
'statsd_port': 1234
}
t.assert_equal(statsd.host, "localhost")
t.assert_equal(statsd.port, 8125)
initialize(**options)
t.assert_equal(statsd.host, "myhost")
t.assert_equal(statsd.port, 1234)
def test_set(self):
self.statsd.set('set', 123)
assert self.recv() == 'set:123|s'
def test_gauge(self):
self.statsd.gauge('gauge', 123.4)
assert self.recv() == 'gauge:123.4|g'
def test_counter(self):
self.statsd.increment('page.views')
t.assert_equal('page.views:1|c', self.recv())
self.statsd.increment('page.views', 11)
t.assert_equal('page.views:11|c', self.recv())
self.statsd.decrement('page.views')
t.assert_equal('page.views:-1|c', self.recv())
self.statsd.decrement('page.views', 12)
t.assert_equal('page.views:-12|c', self.recv())
def test_histogram(self):
self.statsd.histogram('histo', 123.4)
t.assert_equal('histo:123.4|h', self.recv())
def test_tagged_gauge(self):
self.statsd.gauge('gt', 123.4, tags=['country:china', 'age:45', 'blue'])
t.assert_equal('gt:123.4|g|#country:china,age:45,blue', self.recv())
def test_tagged_counter(self):
self.statsd.increment('ct', tags=['country:canada', 'red'])
t.assert_equal('ct:1|c|#country:canada,red', self.recv())
def test_tagged_histogram(self):
self.statsd.histogram('h', 1, tags=['red'])
t.assert_equal('h:1|h|#red', self.recv())
def test_sample_rate(self):
self.statsd.increment('c', sample_rate=0)
assert not self.recv()
for i in range(10000):
self.statsd.increment('sampled_counter', sample_rate=0.3)
self.assert_almost_equal(3000, len(self.statsd.socket.payloads), 150)
t.assert_equal('sampled_counter:1|c|@0.3', self.recv())
def test_tags_and_samples(self):
for i in range(100):
self.statsd.gauge('gst', 23, tags=["sampled"], sample_rate=0.9)
def test_tags_and_samples(self):
for i in range(100):
self.statsd.gauge('gst', 23, tags=["sampled"], sample_rate=0.9)
t.assert_equal('gst:23|g|@0.9|#sampled')
def test_timing(self):
self.statsd.timing('t', 123)
t.assert_equal('t:123|ms', self.recv())
def test_event(self):
self.statsd.event('Title', u'L1\nL2', priority='low', date_happened=1375296969)
t.assert_equal(u'_e{5,6}:Title|L1\\nL2|d:1375296969|p:low', self.recv())
self.statsd.event('Title', u'♬ †øU †øU ¥ºu T0µ ♪',
aggregation_key='key', tags=['t1', 't2:v2'])
t.assert_equal(u'_e{5,19}:Title|♬ †øU †øU ¥ºu T0µ ♪|k:key|#t1,t2:v2', self.recv())
def test_service_check(self):
now = int(time.time())
self.statsd.service_check(
'my_check.name', self.statsd.WARNING,
tags=['key1:val1', 'key2:val2'], timestamp=now,
hostname='i-abcd1234', message=u"♬ †øU \n†øU ¥ºu|m: T0µ ♪")
t.assert_equal(
u'_sc|my_check.name|{0}|d:{1}|h:i-abcd1234|#key1:val1,key2:val2|m:{2}'
.format(self.statsd.WARNING, now, u"♬ †øU \\n†øU ¥ºu|m\: T0µ ♪"), self.recv())
@staticmethod
def assert_almost_equal(a, b, delta):
assert 0 <= abs(a - b) <= delta, "%s - %s not within %s" % (a, b, delta)
def test_socket_error(self):
self.statsd.socket = BrokenSocket()
self.statsd.gauge('no error', 1)
assert True, 'success'
def test_timed(self):
@self.statsd.timed('timed.test')
def func(a, b, c=1, d=1):
"""docstring"""
time.sleep(0.5)
return (a, b, c, d)
t.assert_equal('func', func.__name__)
t.assert_equal('docstring', func.__doc__)
result = func(1, 2, d=3)
# Assert it handles args and kwargs correctly.
t.assert_equal(result, (1, 2, 1, 3))
packet = self.recv()
name_value, type_ = packet.split('|')
name, value = name_value.split(':')
t.assert_equal('ms', type_)
t.assert_equal('timed.test', name)
self.assert_almost_equal(0.5, float(value), 0.1)
def test_batched(self):
self.statsd.open_buffer()
self.statsd.gauge('page.views', 123)
self.statsd.timing('timer', 123)
self.statsd.close_buffer()
t.assert_equal('page.views:123|g\ntimer:123|ms', self.recv())
def test_context_manager(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
statsd.gauge('page.views', 123)
statsd.timing('timer', 123)
t.assert_equal('page.views:123|g\ntimer:123|ms', fake_socket.recv())
def test_batched_buffer_autoflush(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
for i in range(51):
statsd.increment('mycounter')
t.assert_equal('\n'.join(['mycounter:1|c' for i in range(50)]), fake_socket.recv())
t.assert_equal('mycounter:1|c', fake_socket.recv())
def test_module_level_instance(self):
t.assert_true(isinstance(statsd, DogStatsd))
def test_instantiating_does_not_connect(self):
dogpound = DogStatsd()
t.assert_equal(None, dogpound.socket)
def test_accessing_socket_opens_socket(self):
dogpound = DogStatsd()
try:
t.assert_not_equal(None, dogpound.get_socket())
finally:
dogpound.socket.close()
def test_accessing_socket_multiple_times_returns_same_socket(self):
dogpound = DogStatsd()
fresh_socket = FakeSocket()
dogpound.socket = fresh_socket
t.assert_equal(fresh_socket, dogpound.get_socket())
t.assert_not_equal(FakeSocket(), dogpound.get_socket())
class ProxySQLMetrics:
def __init__(self):
self.dogstatsd = DogStatsd(host=os.environ.get('DATADOG_HOST'),
port=os.environ.get('DATADOG_PORT'))
def check(self, instance):
host, port, user, password, tags, options, connect_timeout = self._get_config(
instance)
if not host or not port or not user or not password:
raise Exception(
"ProxySQL host, port, user and password are needed")
with self._connect(host, port, user, password,
connect_timeout) as conn:
try:
# Metric Collection
self._collect_metrics(conn, tags, options)
except Exception as e:
logger.error("ProxySQL collect metrics error: %s" % e,
exc_info=True)
raise e
def _collect_metrics(self, conn, tags, options):
"""Collects all the different types of ProxySQL metrics and submits them to Datadog"""
global_stats = self._get_global_stats(conn)
if global_stats:
for proxysql_metric_name, metric_details in PROXYSQL_MYSQL_STATS_GLOBAL.items(
):
metric_name, metric_type = metric_details
metric_tags = list(tags)
self._submit_metric(
metric_name, metric_type,
float(global_stats.get(proxysql_metric_name)), metric_tags)
report_command_counters = options.get('extra_command_counter_metrics',
True)
if report_command_counters:
command_counters = self._get_command_counters(conn)
for proxysql_metric_name, metric_details in PROXYSQL_MYSQL_STATS_COMMAND_COUNTERS.items(
):
metric_name, metric_type = metric_details
metric_tags = list(tags)
self._submit_metric(
metric_name, metric_type,
float(command_counters.get(proxysql_metric_name)),
metric_tags)
report_conn_pool_stats = options.get('extra_connection_pool_metrics',
True)
if report_conn_pool_stats:
conn_pool_stats = self._get_connection_pool_stats(conn)
for proxysql_metric_name, metric_details in PROXYSQL_CONNECTION_POOL_STATS.items(
):
metric_name, metric_type = metric_details
for metric in conn_pool_stats.get(proxysql_metric_name):
metric_tags = list(tags)
tag, value = metric
if tag:
metric_tags.append(tag)
self._submit_metric(metric_name, metric_type, float(value),
metric_tags)
def _get_global_stats(self, conn):
"""Fetch the global ProxySQL stats."""
sql = 'SELECT * FROM stats.stats_mysql_global'
try:
with closing(conn.cursor()) as cursor:
cursor.execute(sql)
if cursor.rowcount < 1:
logger.debug(
"Failed to fetch records from the stats schema 'stats_mysql_global' table."
)
return None
return {
row['Variable_Name']: row['Variable_Value']
for row in cursor.fetchall()
}
except (pymysql.err.InternalError, pymysql.err.OperationalError) as e:
logger.debug("ProxySQL global stats unavailable at this time: %s" %
str(e))
return None
def _get_command_counters(self, conn):
"""Fetch ProxySQL stats per command type."""
sql = ('SELECT SUM(Total_Time_us) AS query_sum_time_us, '
'SUM(Total_cnt) AS query_count '
'FROM stats.stats_mysql_commands_counters')
try:
with closing(conn.cursor()) as cursor:
cursor.execute(sql)
if cursor.rowcount < 1:
logger.debug(
"Failed to fetch records from the stats schema 'stats_mysql_commands_counters' table."
)
return None
row = cursor.fetchone()
return {
'Query_sum_time': row['query_sum_time_us'],
'Query_count': row['query_count']
}
except (pymysql.err.InternalError, pymysql.err.OperationalError) as e:
logger.debug(
"ProxySQL commands_counters stats unavailable at this time: %s"
% str(e))
return None
def _get_connection_pool_stats(self, conn):
"""Fetch ProxySQL connection pool stats"""
sql = 'SELECT * FROM stats_mysql_connection_pool'
try:
with closing(conn.cursor()) as cursor:
cursor.execute(sql)
if cursor.rowcount < 1:
logger.debug(
"Failed to fetch records from the stats schema 'stats_mysql_commands_counters' table."
)
return None
stats = defaultdict(list)
for row in cursor.fetchall():
stats['Connections_used'].append(
('proxysql_db_node:%s' % row['srv_host'],
row['ConnUsed']))
stats['Connections_free'].append(
('proxysql_db_node:%s' % row['srv_host'],
row['ConnFree']))
stats['Connections_ok'].append(
('proxysql_db_node:%s' % row['srv_host'],
row['ConnOK']))
stats['Connections_error'].append(
('proxysql_db_node:%s' % row['srv_host'],
row['ConnERR']))
stats['Queries'].append(
('proxysql_db_node:%s' % row['srv_host'],
row['Queries']))
stats['Bytes_data_sent'].append(
('proxysql_db_node:%s' % row['srv_host'],
row['Bytes_data_sent']))
stats['Bytes_data_recv'].append(
('proxysql_db_node:%s' % row['srv_host'],
row['Bytes_data_recv']))
# https://github.com/sysown/proxysql/issues/882
# Latency_ms was actually returning values in microseconds
# ProxySQL v1.3.3 returns it with the correct key 'Latency_us'
latency_key = 'Latency_ms' if row.get(
'Latency_ms') else 'Latency_us'
stats['Latency_ms'].append(
('proxysql_db_node:%s' % row['srv_host'],
str(int(row[latency_key]) / 1000)))
return stats
except (pymysql.err.InternalError, pymysql.err.OperationalError) as e:
logger.debug(
"ProxySQL commands_counters stats unavailable at this time: %s"
% str(e))
return None
@staticmethod
def _get_config(instance):
host = instance.get('server', '')
port = int(instance.get('port', 0))
user = instance.get('user', '')
password = str(instance.get('pass', ''))
tags = instance.get('tags', [])
options = instance.get('options', {})
connect_timeout = instance.get('connect_timeout', None)
return host, port, user, password, tags, options, connect_timeout
@contextmanager
def _connect(self, host, port, user, password, connect_timeout):
db = None
try:
db = pymysql.connect(host=host,
port=port,
user=user,
passwd=password,
connect_timeout=connect_timeout,
cursorclass=pymysql.cursors.DictCursor)
logger.debug("Connected to ProxySQL")
yield db
except Exception:
raise
finally:
if db:
db.close()
def _submit_metric(self, metric_name, metric_type, metric_value,
metric_tags):
logger.debug(u"Submitting metric: {}, {}, {}, {}".format(
metric_name, metric_type, metric_value, metric_tags))
if metric_value is None:
return
if metric_type == RATE:
logger.debug(u"Submitted")
self.dogstatsd.increment(metric_name,
metric_value,
tags=metric_tags)
elif metric_type == GAUGE:
logger.debug(u"Submitted")
self.dogstatsd.gauge(metric_name, metric_value, tags=metric_tags)
class StatsD(object):
def __init__(self, app=None, config=None):
"""
Constructor for `flask.ext.datadog.StatsD`
>>> from flask.ext.datadog import StatsD
>>> app = Flask(__name__)
>>> statsd = StatsD(app=app)
:param app: Flask app to configure this client for, if `app` is `None`, then do not
configure yet (call `init_app` manually instead)
:type app: flask.Flask or None
:param config: Configuration for this client to use instead of `app.config`
:type config: dict or None
"""
self.config = config
self.statsd = None
# If an app was provided, then call `init_app` for them
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app, config=None):
"""
Initialize Datadog DogStatsd client from Flask app
>>> from flask.ext.datadog import StatsD
>>> app = Flask(__name__)
>>> statsd = StatsD()
>>> statsd.init_app(app=app)
Available DogStatsd config settings:
STATSD_HOST - statsd host to send metrics to (default: 'localhost')
STATSD_MAX_BUFFER_SIZE - max number of metrics to buffer before sending, only used when batching (default: 50)
STATSD_NAMESPACE - metric name prefix to use, e.g. 'app_name' (default: None)
STATSD_PORT - statsd port to send metrics to (default: 8125)
STATSD_TAGS - list of tags to include by default, e.g. ['env:prod'] (default: None)
STATSD_USEMS - whether or not to report timing in milliseconds (default: False)
Available Flask-Datadog config settings:
DATADOG_CONFIGURE_MIDDLEWARE - whether or not to setup response timing middleware (default: True)
DATADOG_RESPONSE_METRIC_NAME - the name of the response time metric (default: 'flask.response.time')
DATADOG_RESPONSE_SAMPLE_RATE - the sample rate to use for response timing middleware (default: 1)
DATADOG_RESPONSE_AUTO_TAG - whether to auto-add request/response tags to response metrics (default: True)
DATADOG_RESPONSE_ENDPOINT_TAG_NAME - tag name to use for request endpoint tag name (default: 'endpoint')
DATADOG_RESPONSE_METHOD_TAG_NAME - tag name to use for the request method tag name (default: 'method')
:param app: Flask app to configure this client for
:type app: flask.Flask
:param config: optional, dictionary of config values (defaults to `app.config`)
:type config: dict
"""
# Used passed in config if provided, otherwise use the config from `app`
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
# Set default values for expected config properties
self.config.setdefault('STATSD_HOST', 'localhost')
self.config.setdefault('STATSD_MAX_BUFFER_SIZE', 50)
self.config.setdefault('STATSD_NAMESPACE', None)
self.config.setdefault('STATSD_PORT', 8125)
self.config.setdefault('STATSD_TAGS', None)
self.config.setdefault('STATSD_USEMS', False)
self.app = app
# Configure DogStatsd client
# https://github.com/DataDog/datadogpy/blob/v0.11.0/datadog/dogstatsd/base.py
self.statsd = DogStatsd(host=self.config['STATSD_HOST'],
port=self.config['STATSD_PORT'],
max_buffer_size=self.config['STATSD_MAX_BUFFER_SIZE'],
namespace=self.config['STATSD_NAMESPACE'],
constant_tags=self.config['STATSD_TAGS'],
use_ms=self.config['STATSD_USEMS'])
# Configure any of our middleware
self.setup_middleware()
def timer(self, *args, **kwargs):
"""Helper to get a `flask_datadog.TimerWrapper` for this `DogStatsd` client"""
return TimerWrapper(self.statsd, *args, **kwargs)
def incr(self, *args, **kwargs):
"""Helper to expose `self.statsd.increment` under a shorter name"""
return self.statsd.increment(*args, **kwargs)
def decr(self, *args, **kwargs):
"""Helper to expose `self.statsd.decrement` under a shorter name"""
return self.statsd.decrement(*args, **kwargs)
def setup_middleware(self):
"""Helper to configure/setup any Flask-Datadog middleware"""
# Configure response time middleware (if desired)
self.config.setdefault('DATADOG_CONFIGURE_MIDDLEWARE', True)
self.config.setdefault('DATADOG_RESPONSE_METRIC_NAME', 'flask.response.time')
self.config.setdefault('DATADOG_RESPONSE_SAMPLE_RATE', 1)
self.config.setdefault('DATADOG_RESPONSE_AUTO_TAG', True)
self.config.setdefault('DATADOG_RESPONSE_ENDPOINT_TAG_NAME', 'endpoint')
self.config.setdefault('DATADOG_RESPONSE_METHOD_TAG_NAME', 'method')
if self.config['DATADOG_CONFIGURE_MIDDLEWARE']:
self.app.before_request(self.before_request)
self.app.after_request(self.after_request)
def before_request(self):
"""
Flask-Datadog middleware handle for before each request
"""
# Set the request start time
g.flask_datadog_start_time = time.time()
g.flask_datadog_request_tags = []
# Add some default request tags
if self.config['DATADOG_RESPONSE_AUTO_TAG']:
self.add_request_tags([
# Endpoint tag
'{tag_name}:{endpoint}'.format(tag_name=self.config['DATADOG_RESPONSE_ENDPOINT_TAG_NAME'],
endpoint=str(request.endpoint).lower()),
# Method tag
'{tag_name}:{method}'.format(tag_name=self.config['DATADOG_RESPONSE_METHOD_TAG_NAME'],
method=request.method.lower()),
])
def after_request(self, response):
"""
Flask-Datadog middleware handler for after each request
:param response: the response to be sent to the client
:type response: ``flask.Response``
:rtype: ``flask.Response``
"""
# Return early if we don't have the start time
if not hasattr(g, 'flask_datadog_start_time'):
return response
# Get the response time for this request
elapsed = time.time() - g.flask_datadog_start_time
# Convert the elapsed time to milliseconds if they want them
if self.use_ms:
elapsed = int(round(1000 * elapsed))
# Add some additional response tags
if self.config['DATADOG_RESPONSE_AUTO_TAG']:
self.add_request_tags(['status_code:%s' % (response.status_code, )])
# Emit our timing metric
self.statsd.timing(self.config['DATADOG_RESPONSE_METRIC_NAME'],
elapsed,
self.get_request_tags(),
self.config['DATADOG_RESPONSE_SAMPLE_RATE'])
# We ALWAYS have to return the original response
return response
def get_request_tags(self):
"""
Get the current list of tags set for this request
:rtype: list
"""
return getattr(g, 'flask_datadog_request_tags', [])
def add_request_tags(self, tags):
"""
Add the provided list of tags to the tags stored for this request
:param tags: tags to add to this requests tags
:type tags: list
:rtype: list
"""
# Get the current list of tags to append to
# DEV: We use this method since ``self.get_request_tags`` will ensure that we get a list back
current_tags = self.get_request_tags()
# Append our new tags, and return the new full list of tags for this request
g.flask_datadog_request_tags = current_tags + tags
return g.flask_datadog_request_tags
def __getattr__(self, name):
"""
Magic method for fetching any underlying attributes from `self.statsd`
We utilize `__getattr__` to ensure that we are always compatible with
the `DogStatsd` client.
"""
# If `self.statsd` has the attribute then return that attribute
if self.statsd and hasattr(self.statsd, name):
return getattr(self.statsd, name)
raise AttributeError('\'StatsD\' has has attribute \'{name}\''.format(name=name))
def __enter__(self):
"""
Helper to expose the underlying `DogStatsd` client for context managing
>>> statsd = StatsD(app=app)
>>> # Batch any metrics within the `with` block
>>> with statsd:
>>> statsd.increment('metric')
"""
return self.statsd.__enter__()
def __exit__(self, *args, **kwargs):
"""Helper to expose the underlying `DogStatsd` client for context managing"""
return self.statsd.__exit__(*args, **kwargs)
class StatsD(object):
def __init__(self, app=None, config=None):
"""
Constructor for `flask.ext.datadog.StatsD`
>>> from flask.ext.datadog import StatsD
>>> app = Flask(__name__)
>>> statsd = StatsD(app=app)
:param app: Flask app to configure this client for, if `app` is `None`, then do not
configure yet (call `init_app` manually instead)
:type app: flask.Flask or None
:param config: Configuration for this client to use instead of `app.config`
:type config: dict or None
"""
self.config = config
self.statsd = None
# If an app was provided, then call `init_app` for them
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app, config=None):
"""
Initialize Datadog DogStatsd client from Flask app
>>> from flask.ext.datadog import StatsD
>>> app = Flask(__name__)
>>> statsd = StatsD()
>>> statsd.init_app(app=app)
Available DogStatsd config settings:
STATSD_HOST - statsd host to send metrics to (default: 'localhost')
STATSD_MAX_BUFFER_SIZE - max number of metrics to buffer before sending, only used when batching (default: 50)
STATSD_NAMESPACE - metric name prefix to use, e.g. 'app_name' (default: None)
STATSD_PORT - statsd port to send metrics to (default: 8125)
STATSD_TAGS - list of tags to include by default, e.g. ['env:prod'] (default: None)
STATSD_USEMS - whether or not to report timing in milliseconds (default: False)
Available Flask-Datadog config settings:
DATADOG_CONFIGURE_MIDDLEWARE - whether or not to setup response timing middleware (default: True)
DATADOG_RESPONSE_METRIC_NAME - the name of the response time metric (default: 'flask.response.time')
DATADOG_RESPONSE_SIZE_METRIC_NAME - the name of the response time metric (default: 'flask.response.size')
DATADOG_RESPONSE_SAMPLE_RATE - the sample rate to use for response timing middleware (default: 1)
DATADOG_RESPONSE_AUTO_TAG - whether to auto-add request/response tags to response metrics (default: True)
DATADOG_RESPONSE_ENDPOINT_TAG_NAME - tag name to use for request endpoint tag name (default: 'endpoint')
DATADOG_RESPONSE_METHOD_TAG_NAME - tag name to use for the request method tag name (default: 'method')
:param app: Flask app to configure this client for
:type app: flask.Flask
:param config: optional, dictionary of config values (defaults to `app.config`)
:type config: dict
"""
# Used passed in config if provided, otherwise use the config from `app`
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
# Set default values for expected config properties
self.config.setdefault('STATSD_HOST', 'localhost')
self.config.setdefault('STATSD_MAX_BUFFER_SIZE', 50)
self.config.setdefault('STATSD_NAMESPACE', None)
self.config.setdefault('STATSD_PORT', 8125)
self.config.setdefault('STATSD_TAGS', None)
self.config.setdefault('STATSD_USEMS', False)
self.app = app
# Configure DogStatsd client
# https://github.com/DataDog/datadogpy/blob/v0.11.0/datadog/dogstatsd/base.py
self.statsd = DogStatsd(host=self.config['STATSD_HOST'],
port=self.config['STATSD_PORT'],
max_buffer_size=self.config['STATSD_MAX_BUFFER_SIZE'],
namespace=self.config['STATSD_NAMESPACE'],
constant_tags=self.config['STATSD_TAGS'],
use_ms=self.config['STATSD_USEMS'])
# Configure any of our middleware
self.setup_middleware()
def timer(self, *args, **kwargs):
"""Helper to get a `flask_datadog.TimerWrapper` for this `DogStatsd` client"""
return TimerWrapper(self.statsd, *args, **kwargs)
def incr(self, *args, **kwargs):
"""Helper to expose `self.statsd.increment` under a shorter name"""
return self.statsd.increment(*args, **kwargs)
def decr(self, *args, **kwargs):
"""Helper to expose `self.statsd.decrement` under a shorter name"""
return self.statsd.decrement(*args, **kwargs)
def setup_middleware(self):
"""Helper to configure/setup any Flask-Datadog middleware"""
# Configure response time middleware (if desired)
self.config.setdefault('DATADOG_CONFIGURE_MIDDLEWARE', True)
self.config.setdefault('DATADOG_RESPONSE_SIZE_METRIC_NAME', 'flask.response.size')
self.config.setdefault('DATADOG_RESPONSE_METRIC_NAME', 'flask.response.time')
self.config.setdefault('DATADOG_RESPONSE_SAMPLE_RATE', 1)
self.config.setdefault('DATADOG_RESPONSE_AUTO_TAG', True)
self.config.setdefault('DATADOG_RESPONSE_ENDPOINT_TAG_NAME', 'endpoint')
self.config.setdefault('DATADOG_RESPONSE_METHOD_TAG_NAME', 'method')
if self.config['DATADOG_CONFIGURE_MIDDLEWARE']:
self.app.before_request(self.before_request)
self.app.after_request(self.after_request)
def before_request(self):
"""
Flask-Datadog middleware handle for before each request
"""
# Set the request start time
g.flask_datadog_start_time = time.time()
g.flask_datadog_request_tags = []
# Add some default request tags
if self.config['DATADOG_RESPONSE_AUTO_TAG']:
self.add_request_tags([
# Endpoint tag
'{tag_name}:{endpoint}'.format(tag_name=self.config['DATADOG_RESPONSE_ENDPOINT_TAG_NAME'],
endpoint=str(request.endpoint).lower()),
# Method tag
'{tag_name}:{method}'.format(tag_name=self.config['DATADOG_RESPONSE_METHOD_TAG_NAME'],
method=request.method.lower()),
])
def after_request(self, response):
"""
Flask-Datadog middleware handler for after each request
:param response: the response to be sent to the client
:type response: ``flask.Response``
:rtype: ``flask.Response``
"""
# Return early if we don't have the start time
if not hasattr(g, 'flask_datadog_start_time'):
return response
# Get the response time for this request
elapsed = time.time() - g.flask_datadog_start_time
# Convert the elapsed time to milliseconds if they want them
if self.use_ms:
elapsed = int(round(1000 * elapsed))
# Add some additional response tags
if self.config['DATADOG_RESPONSE_AUTO_TAG']:
self.add_request_tags(['status_code:%s' % (response.status_code, )])
tags = self.get_request_tags()
sample_rate = self.config['DATADOG_RESPONSE_SAMPLE_RATE']
# Emit timing metric
self.statsd.timing(self.config['DATADOG_RESPONSE_METRIC_NAME'],
elapsed,
tags,
sample_rate)
# Emit response size metric
if 'content-length' in response.headers:
size = int(response.headers['content-length'])
self.statsd.histogram(self.config['DATADOG_RESPONSE_SIZE_METRIC_NAME'],
size,
tags,
sample_rate)
# We ALWAYS have to return the original response
return response
def get_request_tags(self):
"""
Get the current list of tags set for this request
:rtype: list
"""
return getattr(g, 'flask_datadog_request_tags', [])
def add_request_tags(self, tags):
"""
Add the provided list of tags to the tags stored for this request
:param tags: tags to add to this requests tags
:type tags: list
:rtype: list
"""
# Get the current list of tags to append to
# DEV: We use this method since ``self.get_request_tags`` will ensure that we get a list back
current_tags = self.get_request_tags()
# Append our new tags, and return the new full list of tags for this request
g.flask_datadog_request_tags = current_tags + tags
return g.flask_datadog_request_tags
def __getattr__(self, name):
"""
Magic method for fetching any underlying attributes from `self.statsd`
We utilize `__getattr__` to ensure that we are always compatible with
the `DogStatsd` client.
"""
# If `self.statsd` has the attribute then return that attribute
if self.statsd and hasattr(self.statsd, name):
return getattr(self.statsd, name)
raise AttributeError('\'StatsD\' has has attribute \'{name}\''.format(name=name))
def __enter__(self):
"""
Helper to expose the underlying `DogStatsd` client for context managing
>>> statsd = StatsD(app=app)
>>> # Batch any metrics within the `with` block
>>> with statsd:
>>> statsd.increment('metric')
"""
return self.statsd.__enter__()
def __exit__(self, *args, **kwargs):
"""Helper to expose the underlying `DogStatsd` client for context managing"""
return self.statsd.__exit__(*args, **kwargs)
