def test_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
statsd = DogStatsd(constant_tags=['country:canada', 'red'])
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal('gt:123.4|g|#country:canada,red,country:china,age:45,blue', statsd.socket.recv())
def test_telemetry_flush_interval_alternate_destination(self):
statsd = DogStatsd(telemetry_host='foo')
fake_socket = FakeSocket()
statsd.socket = fake_socket
fake_telemetry_socket = FakeSocket()
statsd.telemetry_socket = fake_telemetry_socket
assert statsd.telemetry_host != None
assert statsd.telemetry_port != None
assert statsd._dedicated_telemetry_destination()
# set the last flush time in the future to be sure we won't flush
statsd._last_flush_time = time.time(
) + statsd._telemetry_flush_interval
statsd.gauge('gauge', 123.4)
assert_equal('gauge:123.4|g', fake_socket.recv())
t1 = time.time()
# setting the last flush time in the past to trigger a telemetry flush
statsd._last_flush_time = t1 - statsd._telemetry_flush_interval - 1
statsd.gauge('gauge', 123.4)
assert_equal('gauge:123.4|g', fake_socket.recv())
assert_equal_telemetry('',
fake_telemetry_socket.recv(),
telemetry=telemetry_metrics(metrics=2,
bytes_sent=13 * 2,
packets_sent=2))
# assert that _last_flush_time has been updated
assert t1 < statsd._last_flush_time
def test_batched_buffer_autoflush(self):
fake_socket = FakeSocket()
bytes_sent = 0
with DogStatsd(telemetry_min_flush_interval=0) as dogstatsd:
single_metric = 'mycounter:1|c'
self.assertEqual(dogstatsd._max_payload_size,
UDP_OPTIMAL_PAYLOAD_LENGTH)
metrics_per_packet = dogstatsd._max_payload_size // (
len(single_metric) + 1)
dogstatsd.socket = fake_socket
for _ in range(metrics_per_packet + 1):
dogstatsd.increment('mycounter')
payload = '\n'.join(
[single_metric for _ in range(metrics_per_packet)])
telemetry = telemetry_metrics(metrics=metrics_per_packet + 1,
bytes_sent=len(payload))
bytes_sent += len(payload) + len(telemetry)
self.assertEqual(payload, fake_socket.recv())
self.assertEqual(telemetry, fake_socket.recv())
self.assertEqual(single_metric, fake_socket.recv())
telemetry = telemetry_metrics(metrics=0,
packets_sent=2,
bytes_sent=len(single_metric) +
len(telemetry))
self.assertEqual(telemetry, fake_socket.recv())
def init_statsd_client(statsd_host='localhost', statsd_port=8125):
global statsd
if not statsd:
statsd = DogStatsd(statsd_host,
statsd_port,
use_ms=True,
namespace=METRIC_PREFIX)
def test_telemetry_flush_interval(self):
dogstatsd = DogStatsd()
fake_socket = FakeSocket()
dogstatsd.socket = fake_socket
# set the last flush time in the future to be sure we won't flush
dogstatsd._last_flush_time = time.time(
) + dogstatsd._telemetry_flush_interval
dogstatsd.gauge('gauge', 123.4)
metric = 'gauge:123.4|g'
self.assertEqual(metric, fake_socket.recv())
time1 = time.time()
# setting the last flush time in the past to trigger a telemetry flush
dogstatsd._last_flush_time = time1 - dogstatsd._telemetry_flush_interval - 1
dogstatsd.gauge('gauge', 123.4)
self.assert_equal_telemetry(metric,
fake_socket.recv(2),
telemetry=telemetry_metrics(
metrics=2,
bytes_sent=2 * len(metric),
packets_sent=2))
# assert that _last_flush_time has been updated
self.assertTrue(time1 < dogstatsd._last_flush_time)
def get_velibs():
r = requests.get(
"https://www.velib-metropole.fr/webapi/map/details?gpsTopLatitude=48.85240750385378&gpsTopLongitude=2.378431586815509&gpsBotLatitude=48.850687689726385&gpsBotLongitude=2.3723489091318015&zoomLevel=17.174858916253687"
)
velib_stations = json.loads(r.content.decode("utf-8"))
statsd = DogStatsd(host=HOST, port=PORT, constant_tags=["host:api"])
for station in velib_stations:
tag = get_tag(station)
bike_nb = station["nbBike"]
ebike_nb = station["nbEbike"]
free_dock_nb = station["nbFreeDock"] + station["nbFreeEDock"]
dock_nb = station["nbDock"] + station["nbEDock"]
statsd.gauge("velib.regular_bikes", bike_nb, tags=tag)
statsd.gauge("velib.electric_bikes", ebike_nb, tags=tag)
statsd.gauge("velib.free_docks", free_dock_nb, tags=tag)
statsd.gauge("velib.total_docks", dock_nb, tags=tag)
statsd.service_check(
"velib.station_state",
get_state(station),
tags=tag,
message=station["station"]["state"],
)
logger.info(
"Found {} ebike(s), {} bike(s), {} free dock(s) on {} dock(s). The station is {}"
.format(ebike_nb, bike_nb, free_dock_nb, dock_nb,
station["station"]["state"]),
extra={"station": get_station_name(station)},
)
def test_telemetry_flush_interval_alternate_destination(self):
dogstatsd = DogStatsd(telemetry_host='foo')
fake_socket = FakeSocket()
dogstatsd.socket = fake_socket
fake_telemetry_socket = FakeSocket()
dogstatsd.telemetry_socket = fake_telemetry_socket
self.assertIsNotNone(dogstatsd.telemetry_host)
self.assertIsNotNone(dogstatsd.telemetry_port)
self.assertTrue(dogstatsd._dedicated_telemetry_destination())
# set the last flush time in the future to be sure we won't flush
dogstatsd._last_flush_time = time.time(
) + dogstatsd._telemetry_flush_interval
dogstatsd.gauge('gauge', 123.4)
self.assertEqual('gauge:123.4|g', fake_socket.recv())
time1 = time.time()
# setting the last flush time in the past to trigger a telemetry flush
dogstatsd._last_flush_time = time1 - dogstatsd._telemetry_flush_interval - 1
dogstatsd.gauge('gauge', 123.4)
self.assertEqual('gauge:123.4|g', fake_socket.recv())
self.assert_equal_telemetry('',
fake_telemetry_socket.recv(),
telemetry=telemetry_metrics(
metrics=2,
bytes_sent=13 * 2,
packets_sent=2))
# assert that _last_flush_time has been updated
self.assertTrue(time1 < dogstatsd._last_flush_time)
def test_telemetry_flush_interval(self):
statsd = DogStatsd()
fake_socket = FakeSocket()
statsd.socket = fake_socket
# set the last flush time in the future to be sure we won't flush
statsd._last_flush_time = time.time(
) + statsd._telemetry_flush_interval
statsd.gauge('gauge', 123.4)
metric = 'gauge:123.4|g'
assert_equal(metric, fake_socket.recv())
t1 = time.time()
# setting the last flush time in the past to trigger a telemetry flush
statsd._last_flush_time = t1 - statsd._telemetry_flush_interval - 1
statsd.gauge('gauge', 123.4)
assert_equal_telemetry(metric,
fake_socket.recv(2),
telemetry=telemetry_metrics(metrics=2,
bytes_sent=2 *
len(metric),
packets_sent=2))
# assert that _last_flush_time has been updated
assert t1 < statsd._last_flush_time
def test_default_route(self):
"""
Dogstatsd host can be dynamically set to the default route.
"""
self.assertEqual(
DogStatsd(use_default_route=True).host,
"172.17.0.1"
)
def test_context_manager(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
statsd.gauge('page.views', 123)
statsd.timing('timer', 123)
t.assert_equal('page.views:123|g\ntimer:123|ms', fake_socket.recv())
def test_entity_tag_from_environment_and_constant(self):
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ['DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
statsd = DogStatsd(constant_tags=['country:canada', 'red'])
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
t.assert_equal(
'gt:123.4|g|#country:canada,red,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d',
statsd.socket.recv())
def test_batched_buffer_autoflush(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
for i in range(51):
statsd.increment('mycounter')
t.assert_equal('\n'.join(['mycounter:1|c' for i in range(50)]), fake_socket.recv())
t.assert_equal('mycounter:1|c', fake_socket.recv())
def test_entity_tag_from_environment(self):
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ['DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
statsd = DogStatsd()
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
assert_equal(
'gt:123.4|g|#dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d',
statsd.socket.recv())
def test_default_route(self):
"""
Dogstatsd host can be dynamically set to the default route.
"""
# Setup
statsd = DogStatsd(use_default_route=True)
# Assert
t.assert_equal(statsd.host, "172.17.0.1")
def test_context_manager(self):
fake_socket = FakeSocket()
with DogStatsd(telemetry_min_flush_interval=0) as dogstatsd:
dogstatsd.socket = fake_socket
dogstatsd.gauge('page.views', 123)
dogstatsd.timing('timer', 123)
metric = "page.views:123|g\ntimer:123|ms"
self.assertEqual(metric, fake_socket.recv())
self.assertEqual(telemetry_metrics(metrics=2, bytes_sent=len(metric)), fake_socket.recv())
def test_tags_from_environment(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
dogstatsd = DogStatsd(telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
dogstatsd.gauge('gt', 123.4)
metric = 'gt:123.4|g|#country:china,age:45,blue'
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(telemetry_metrics(tags="country:china,age:45,blue", bytes_sent=len(metric)), dogstatsd.socket.recv())
def test_tags_from_environment(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
statsd = DogStatsd()
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
assert_equal_telemetry(
'gt:123.4|g|#country:china,age:45,blue',
statsd.socket.recv(),
telemetry=telemetry_metrics(tags="country:china,age:45,blue"))
def test_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
statsd = DogStatsd(constant_tags=['country:canada', 'red'], telemetry_min_flush_interval=0)
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
tags="country:canada,red,country:china,age:45,blue"
metric = 'gt:123.4|g|#'+tags
assert_equal(metric, statsd.socket.recv())
assert_equal(telemetry_metrics(tags=tags, bytes_sent=len(metric)), statsd.socket.recv())
def test_context_manager(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
statsd.gauge('page.views', 123)
statsd.timing('timer', 123)
assert_equal_telemetry("page.views:123|g\ntimer:123|ms",
fake_socket.recv(),
telemetry=telemetry_metrics(metrics=2))
def test_entity_tag_from_environment(self):
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ['DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
dogstatsd = DogStatsd(telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
dogstatsd.gauge('gt', 123.4)
metric = 'gt:123.4|g|#dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d'
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(
telemetry_metrics(tags="dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d", bytes_sent=len(metric)),
dogstatsd.socket.recv())
def test_entity_tag_from_environment_and_constant(self):
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ['DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
statsd = DogStatsd(constant_tags=['country:canada', 'red'], telemetry_min_flush_interval=0)
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
metric = 'gt:123.4|g|#country:canada,red,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d'
assert_equal(metric, statsd.socket.recv())
assert_equal(
telemetry_metrics(tags="country:canada,red,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d",
bytes_sent=len(metric)), statsd.socket.recv())
def setUp(self):
"""
Set up a default Dogstatsd instance and mock the proc filesystem.
"""
#
self.statsd = DogStatsd()
self.statsd.socket = FakeSocket()
# Mock the proc filesystem
route_data = load_fixtures('route')
self._procfs_mock = patch('datadog.util.compat.builtins.open', mock_open())
self._procfs_mock.__enter__().return_value.readlines.return_value = route_data.split("\n")
def test_entity_tag_and_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ['DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
dogstatsd = DogStatsd(constant_tags=['country:canada', 'red'], telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
dogstatsd.gauge('gt', 123.4)
tags = "country:canada,red,country:china,age:45,blue,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d"
metric = 'gt:123.4|g|#'+tags
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(telemetry_metrics(tags=tags, bytes_sent=len(metric)), dogstatsd.socket.recv())
def test_dogstatsd_initialization_with_dd_env_service_version(self):
"""
Dogstatsd should automatically use DD_ENV, DD_SERVICE, and DD_VERSION (if present)
to set {env, service, version} as global tags for all metrics emitted.
"""
cases = [
# Test various permutations of setting DD_* env vars, as well as other global tag configuration.
# An empty string signifies that the env var either isn't set or that it is explicitly set to empty string.
('', '', '', '', [], []),
('prod', '', '', '', [], ['env:prod']),
('prod', 'dog', '', '', [], ['env:prod', 'service:dog']),
('prod', 'dog', 'abc123', '', [], ['env:prod', 'service:dog', 'version:abc123']),
('prod', 'dog', 'abc123', 'env:prod,type:app', [], ['env:prod', 'env:prod', 'service:dog', 'type:app', 'version:abc123']),
('prod', 'dog', 'abc123', 'env:prod2,type:app', [], ['env:prod', 'env:prod2', 'service:dog', 'type:app', 'version:abc123']),
('prod', 'dog', 'abc123', '', ['env:prod', 'type:app'], ['env:prod', 'env:prod', 'service:dog', 'type:app', 'version:abc123']),
('prod', 'dog', 'abc123', '', ['env:prod2', 'type:app'], ['env:prod', 'env:prod2', 'service:dog', 'type:app', 'version:abc123']),
('prod', 'dog', 'abc123', 'env:prod3,custom_tag:cat', ['env:prod2', 'type:app'], ['custom_tag:cat', 'env:prod', 'env:prod2', 'env:prod3', 'service:dog', 'type:app', 'version:abc123']),
]
for case in cases:
dd_env, dd_service, dd_version, datadog_tags, constant_tags, global_tags = case
with EnvVars(
env_vars={
'DATADOG_TAGS': datadog_tags,
'DD_ENV': dd_env,
'DD_SERVICE': dd_service,
'DD_VERSION': dd_version,
}
):
dogstatsd = DogStatsd(constant_tags=constant_tags, telemetry_min_flush_interval=0)
dogstatsd.socket = FakeSocket()
# Guarantee consistent ordering, regardless of insertion order.
dogstatsd.constant_tags.sort()
self.assertEqual(global_tags, dogstatsd.constant_tags)
# Make call with no tags passed; only the globally configured tags will be used.
global_tags_str = ','.join([t for t in global_tags])
dogstatsd.gauge('gt', 123.4)
# Protect against the no tags case.
metric = 'gt:123.4|g|#{}'.format(global_tags_str) if global_tags_str else 'gt:123.4|g'
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(telemetry_metrics(tags=global_tags_str, bytes_sent=len(metric)), dogstatsd.socket.recv())
dogstatsd._reset_telemetry()
# Make another call with local tags passed.
passed_tags = ['env:prod', 'version:def456', 'custom_tag:toad']
all_tags_str = ','.join([t for t in passed_tags + global_tags])
dogstatsd.gauge('gt', 123.4, tags=passed_tags)
metric = 'gt:123.4|g|#{}'.format(all_tags_str)
self.assertEqual(metric, dogstatsd.socket.recv())
self.assertEqual(telemetry_metrics(tags=global_tags_str, bytes_sent=len(metric)), dogstatsd.socket.recv())
def test_entity_tag_and_tags_from_environment_and_constant(self):
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = 'country:china,age:45,blue'
with preserve_environment_variable('DD_ENTITY_ID'):
os.environ[
'DD_ENTITY_ID'] = '04652bb7-19b7-11e9-9cc6-42010a9c016d'
statsd = DogStatsd(constant_tags=['country:canada', 'red'])
statsd.socket = FakeSocket()
statsd.gauge('gt', 123.4)
tags = "country:canada,red,country:china,age:45,blue,dd.internal.entity_id:04652bb7-19b7-11e9-9cc6-42010a9c016d"
assert_equal_telemetry('gt:123.4|g|#' + tags,
statsd.socket.recv(),
telemetry=telemetry_metrics(tags=tags))
def __init__(self, app, config, statsd=None):
self.config = config
for key, value in DEFAULTS.items():
self.config.setdefault(key, value)
self.statsd = DogStatsd(host=self.config['STATSD_HOST'],
port=self.config['STATSD_PORT'],
max_buffer_size=self.config['STATSD_MAX_BUFFER_SIZE'],
namespace=self.config['STATSD_NAMESPACE'],
constant_tags=self.config['STATSD_TAGS'],
use_ms=self.config['STATSD_USEMS']) \
if statsd is None \
else statsd
self.app = app
def test_socket_creation(self):
"""
Socket creation plays well with multiple threads.
"""
# Create a DogStatsd client but no socket
statsd = DogStatsd()
# Submit metrics from different threads to create a socket
threads = []
for value in range(10000):
t = threading.Thread(target=statsd.gauge, args=("foo", value))
threads.append(t)
t.start()
for t in threads:
t.join()
def test_dogstatsd_initialization_with_env_vars(self):
"""
Dogstatsd can retrieve its config from env vars when
not provided in constructor.
"""
# Setup
with preserve_environment_variable('DD_AGENT_HOST'):
os.environ['DD_AGENT_HOST'] = 'myenvvarhost'
with preserve_environment_variable('DD_DOGSTATSD_PORT'):
os.environ['DD_DOGSTATSD_PORT'] = '4321'
dogstatsd = DogStatsd()
# Assert
self.assertEqual(dogstatsd.host, "myenvvarhost")
self.assertEqual(dogstatsd.port, 4321)
def test_send_batch_metrics(self):
"""
Metrics can be buffered, submitted from concurrent threads.
"""
with DogStatsd() as batch_statsd:
# Create a DogStatsd buffer client with a mocked socket
batch_statsd.socket = self.socket
# Samples
values = set(range(10000))
# Submit metrics from different threads
self._submit_with_multiple_threads(batch_statsd, "gauge", values)
# All metrics were properly submitted
self.assertMetrics(values)
def test_timing(self):
"""
Timings can be submitted from concurrent threads.
"""
# Create a DogStatsd client with a mocked socket
statsd = DogStatsd()
statsd.socket = self.socket
# Samples
values = set(range(10000))
# Submit metrics from different threads
self._submit_with_multiple_threads(statsd, "timing", values)
# All metrics were properly submitted
self.assertMetrics(values)
