def test_stats():
# The min is not enabled by default
stats = Stats()
metric_stats = {
'foo': 2,
'bar': 5,
}
# setters
stats.set_stat('metrics', 4)
stats.set_stat('events', 2)
stats.set_stat('service_checks', 1)
# totals
stats.inc_stat('metrics_total', 4)
stats.inc_stat('events_total', 2)
stats.inc_stat('service_checks_total', 1)
# info
stats.set_info('metric_stats', metric_stats)
stats_snapshot, info_snapshot = stats.snapshot()
assert info_snapshot['metric_stats'] == metric_stats
assert stats_snapshot['metrics'] == 4
assert stats_snapshot['events'] == 2
assert stats_snapshot['service_checks'] == 1
assert stats_snapshot['metrics_total'] == 4
assert stats_snapshot['events_total'] == 2
assert stats_snapshot['service_checks_total'] == 1
# test we got a deepcopy for stats
stats.set_stat('metrics', 10)
stats.inc_stat('metrics_total', 10)
assert stats_snapshot != metric_stats
assert stats_snapshot['metrics'] != stats.get_stat('metrics')
# test we got a deepcopy for info
metric_stats['bar'] += 1
stats.set_info('metric_stats', metric_stats)
assert info_snapshot != metric_stats
assert info_snapshot['metric_stats']['foo'] == metric_stats['foo']
assert info_snapshot['metric_stats']['bar'] != metric_stats['bar']
# test for updated snapshots
stats_snapshot, info_snapshot = stats.snapshot()
assert stats_snapshot['metrics'] == 10
assert stats_snapshot['metrics_total'] == 14
assert info_snapshot['metric_stats']['foo'] == metric_stats['foo']
assert info_snapshot['metric_stats']['bar'] == metric_stats['bar']
# test strict get
with pytest.raises(KeyError):
stats.get_stat('nonexistent', strict=True)
with pytest.raises(KeyError):
stats.get_info('nonexistent', strict=True)
class Aggregator(object):
"""
Abstract metric aggregator class.
"""
# Types of metrics that allow strings
ALLOW_STRINGS = ['s', ]
# Types that are not implemented and ignored
IGNORE_TYPES = ['d', ]
# prefixes
SC_PREFIX = '_sc'
EVENT_PREFIX = '_e'
def __init__(self, hostname, interval=1.0, expiry_seconds=300,
formatter=None, recent_point_threshold=None,
histogram_aggregates=None, histogram_percentiles=None,
utf8_decoding=False):
# TODO(jaime): add support for event, service_check sources
self.events = []
self.service_checks = []
self.stats = Stats()
# TODO(jaime): we can probably kill total counts
self.packet_count = 0
self.metric_count = 0
self.event_count = 0
self.service_check_count = 0
self.hostname = hostname
self.expiry_seconds = expiry_seconds
self.formatter = formatter or api_formatter
self.interval = float(interval)
recent_point_threshold = recent_point_threshold or DEFAULT_RECENT_POINT_THRESHOLD
self.recent_point_threshold = int(recent_point_threshold)
self.num_discarded_old_points = 0
# Additional config passed when instantiating metric configs
self.metric_config = {
Histogram: {
'aggregates': histogram_aggregates,
'percentiles': histogram_percentiles
}
}
self.utf8_decoding = utf8_decoding
def deduplicate_tags(self, tags):
return sorted(set(tags))
def packets_per_second(self, interval):
if interval == 0:
return 0
return round(float(self.packet_count)/interval, 2)
def parse_metric_packet(self, packet):
"""
Schema of a dogstatsd packet:
<name>:<value>|<metric_type>|@<sample_rate>|#<tag1_name>:<tag1_value>,<tag2_name>:<tag2_value>:<value>|<metric_type>...
"""
parsed_packets = []
name_and_metadata = packet.split(':', 1)
if len(name_and_metadata) != 2:
raise Exception('Unparseable metric packet: {}'.format(packet))
name = name_and_metadata[0]
broken_split = name_and_metadata[1].split(':')
data = []
partial_datum = None
for token in broken_split:
# We need to fix the tag groups that got broken by the : split
if partial_datum is None:
partial_datum = token
elif "|" not in token:
partial_datum += ":" + token
else:
data.append(partial_datum)
partial_datum = token
data.append(partial_datum)
for datum in data:
value_and_metadata = datum.split('|')
if len(value_and_metadata) < 2:
raise Exception('Unparseable metric packet: {}'.format(packet))
# Submit the metric
raw_value = value_and_metadata[0]
metric_type = value_and_metadata[1]
if metric_type in self.ALLOW_STRINGS:
value = raw_value
elif len(metric_type) > 0 and metric_type[0] in self.IGNORE_TYPES:
continue
else:
# Try to cast as an int first to avoid precision issues, then as a
# float.
try:
value = int(raw_value)
except ValueError:
try:
value = float(raw_value)
except ValueError:
# Otherwise, raise an error saying it must be a number
raise Exception('Metric value must be a number: {}, {}'.format(name, raw_value))
# Parse the optional values - sample rate & tags.
sample_rate = 1
tags = None
try:
for m in value_and_metadata[2:]:
# Parse the sample rate
if m[0] == '@':
sample_rate = float(m[1:])
# in case it's in a bad state
sample_rate = 1 if sample_rate < 0 or sample_rate > 1 else sample_rate
elif m[0] == '#':
tags = tuple(sorted(m[1:].split(',')))
except IndexError:
log.warning('Incorrect metric metadata: metric_name:%s, metadata:%s',
name, ' '.join(value_and_metadata[2:]))
parsed_packets.append((name, value, metric_type, tags, sample_rate))
return parsed_packets
def _unescape_sc_content(self, string):
return string.replace('\\n', '\n').replace(r'm\:', 'm:')
def _unescape_event_text(self, string):
return string.replace('\\n', '\n')
def parse_event_packet(self, packet):
try:
name_and_metadata = packet.split(':', 1)
if len(name_and_metadata) != 2:
raise Exception('Unparseable event packet: {}'.format(packet))
# Event syntax:
# _e{5,4}:title|body|meta
name = name_and_metadata[0]
metadata = name_and_metadata[1]
title_length, text_length = name.split(',')
title_length = int(title_length[3:])
text_length = int(text_length[:-1])
event = {
'title': metadata[:title_length],
'text': self._unescape_event_text(metadata[title_length+1:title_length+text_length+1])
}
meta = metadata[title_length+text_length+1:]
for m in meta.split('|')[1:]:
if m[0] == 't':
event['alert_type'] = m[2:]
elif m[0] == 'k':
event['aggregation_key'] = m[2:]
elif m[0] == 's':
event['source_type_name'] = m[2:]
elif m[0] == 'd':
event['date_happened'] = int(m[2:])
elif m[0] == 'p':
event['priority'] = m[2:]
elif m[0] == 'h':
event['hostname'] = m[2:]
elif m[0] == '#':
event['tags'] = self.deduplicate_tags(m[1:].split(','))
return event
except (IndexError, ValueError):
raise Exception('Unparseable event packet: {}'.format(packet))
def parse_sc_packet(self, packet):
try:
_, data_and_metadata = packet.split('|', 1)
# Service check syntax:
# _sc|check_name|status|meta
if data_and_metadata.count('|') == 1:
# Case with no metadata
check_name, status = data_and_metadata.split('|')
metadata = ''
else:
check_name, status, metadata = data_and_metadata.split('|', 2)
service_check = {
'check_name': check_name,
'status': int(status)
}
message_delimiter = 'm:' if metadata.startswith('m:') else '|m:'
if message_delimiter in metadata:
meta, message = metadata.rsplit(message_delimiter, 1)
service_check['message'] = self._unescape_sc_content(message)
else:
meta = metadata
if not meta:
return service_check
meta = str(meta)
for m in meta.split('|'):
if m[0] == 'd':
service_check['timestamp'] = float(m[2:])
elif m[0] == 'h':
service_check['hostname'] = m[2:]
elif m[0] == '#':
service_check['tags'] = self.deduplicate_tags(m[1:].split(','))
return service_check
except (IndexError, ValueError):
raise Exception('Unparseable service check packet: {}'.format(packet))
def submit_packets(self, packets):
# We should probably consider that packets are always encoded
# in utf8, but decoding all packets has an perf overhead of 7%
# So we let the user decide if we wants utf8 by default
# Keep a very conservative approach anyhow
# Clients MUST always send UTF-8 encoded content
if self.utf8_decoding:
try:
packets = packets.decode('utf-8')
except AttributeError:
pass
for packet in packets.splitlines():
if not packet.strip():
continue
self.packet_count += 1
if packet.startswith(self.EVENT_PREFIX):
event = self.parse_event_packet(packet)
self.event(**event)
self.event_count += 1
elif packet.startswith(self.SC_PREFIX):
service_check = self.parse_sc_packet(packet)
self.service_check(**service_check)
self.service_check_count += 1
else:
parsed_packets = self.parse_metric_packet(packet)
for name, value, mtype, tags, sample_rate in parsed_packets:
hostname, tags = self._extract_magic_tags(tags)
self.submit_metric(name, value, mtype, tags=tags,
hostname=hostname, sample_rate=sample_rate)
def _extract_magic_tags(self, tags):
"""Magic tags (host) override metric hostname attributes"""
hostname = None
# This implementation avoid list operations for the common case
if tags:
tags_to_remove = []
for tag in tags:
if tag.startswith('host:'):
hostname = tag[5:]
tags_to_remove.append(tag)
if tags_to_remove:
# tags is a tuple already sorted, we convert it into a list to pop elements
tags = list(tags)
for tag in tags_to_remove:
tags.remove(tag)
tags = tuple(tags) or None
return hostname, tags
def submit_metric(self, name, value, mtype, tags=None, hostname=None,
timestamp=None, sample_rate=1):
""" Add a metric to be aggregated """
raise NotImplementedError()
def event(self, title, text, date_happened=None, alert_type=None, aggregation_key=None,
source_type_name=None, priority=None, tags=None, hostname=None):
event = {
'msg_title': title,
'msg_text': text,
}
if date_happened is not None:
event['timestamp'] = date_happened
else:
event['timestamp'] = int(time())
if alert_type is not None:
event['alert_type'] = alert_type
if aggregation_key is not None:
event['aggregation_key'] = aggregation_key
if source_type_name is not None:
event['source_type_name'] = source_type_name
if priority is not None:
event['priority'] = priority
if tags is not None:
event['tags'] = self.deduplicate_tags(tags)
if hostname is not None:
event['host'] = hostname
else:
event['host'] = self.hostname
self.events.append(event)
def service_check(self, check_name, status, tags=None, timestamp=None,
hostname=None, message=None):
service_check = {
'check': check_name,
'status': status,
'timestamp': timestamp or int(time())
}
if tags is not None:
service_check['tags'] = self.deduplicate_tags(tags)
if hostname is not None:
service_check['host_name'] = hostname
else:
service_check['host_name'] = self.hostname
if message is not None:
service_check['message'] = message
self.service_checks.append(service_check)
def flush(self):
""" Flush aggregated metrics """
raise NotImplementedError()
def flush_events(self):
events = self.events
self.events = []
self.stats.set_stat('events', self.event_count)
self.stats.inc_stat('events_total', self.event_count)
self.event_count = 0
log.info("Received %d events since last flush", len(events))
return events
def flush_service_checks(self):
service_checks = self.service_checks
self.service_checks = []
self.stats.set_stat('service_checks', self.service_check_count)
self.stats.inc_stat('service_checks_total', self.service_check_count)
self.service_check_count = 0
log.info("Received %d service check runs since last flush", len(service_checks))
return service_checks
def send_packet_count(self, metric_name):
self.submit_metric(metric_name, self.packet_count, 'g')
class Forwarder(object):
V1_ENDPOINT = "/intake/"
V1_SERIES_ENDPOINT = "/api/v1/series"
V1_SERVICE_CHECKS_ENDPOINT = "/api/v1/check_run"
DD_API_HEADER = "DD-Api-Key"
QUEUES_SIZE = 100
WORKER_JOIN_TIME = 2
def __init__(self, api_key, domain, nb_worker=4, proxies={}):
self.api_key = api_key
self.domain = domain
self.stats = Stats()
self.input_queue = queue.Queue(self.QUEUES_SIZE)
self.retry_queue = queue.Queue(self.QUEUES_SIZE)
self.workers = []
self.nb_worker = nb_worker
self.retry_worker = None
self.proxies = proxies
def start(self):
self.retry_worker = RetryWorker(self.input_queue, self.retry_queue, self.stats)
self.retry_worker.start()
for i in range(self.nb_worker):
w = Worker(self.input_queue, self.retry_queue, self.stats)
w.start()
self.workers.append(w)
def stop(self):
self.retry_worker.stop()
for w in self.workers:
w.stop()
self.retry_worker.join(self.WORKER_JOIN_TIME)
if self.retry_worker.is_alive():
log.error("Could not stop thread '%s'", self.retry_worker.name)
self.retry_worker = None
for w in self.workers:
# wait 2 seconds for the worker to stop
w.join(self.WORKER_JOIN_TIME)
if w.is_alive():
log.error("Could not stop thread '%s'", w.name)
self.workers = []
def _submit_payload(self, endpoint, payload, extra_header=None):
endpoint += "?api_key=" + self.api_key
if extra_header:
extra_header[self.DD_API_HEADER] = self.api_key
else:
extra_header = {self.DD_API_HEADER: self.api_key}
t = Transaction(payload, self.domain, endpoint, extra_header, proxies=self.proxies)
try:
self.input_queue.put_nowait(t)
except queue.Full as e:
log.error("Could not submit transaction to '%s', queue is full (dropping it): %s", endpoint, e)
def submit_v1_series(self, payload, extra_header):
self.stats.inc_stat('series_payloads', 1)
self._submit_payload(self.V1_SERIES_ENDPOINT, payload, extra_header)
def submit_v1_intake(self, payload, extra_header):
self.stats.inc_stat('intake_payloads', 1)
self._submit_payload(self.V1_ENDPOINT, payload, extra_header)
def submit_v1_service_checks(self, payload, extra_header):
self.stats.inc_stat('service_check_payloads', 1)
self._submit_payload(self.V1_SERVICE_CHECKS_ENDPOINT, payload, extra_header)
