def test_gauge_flush_during_bucket(self): # Tests returning data when flush is called in the middle of a time bucket that has data ag_interval = self.interval stats = MetricsBucketAggregator('myhost', interval=ag_interval) self.wait_for_bucket_boundary(ag_interval) # Track some counters. stats.submit_packets('my.first.gauge:1|g') stats.submit_packets('my.first.gauge:5|g') stats.submit_packets('my.second.gauge:1.5|g') self.sleep_for_interval_length(ag_interval) stats.submit_packets('my.second.gauge:9.5|g') # Ensure that gauges roll up correctly. metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 2 first, second = metrics assert first['metric'] == 'my.first.gauge' assert first['points'][0][1] == 5 assert first['host'] == 'myhost' assert second['metric'] == 'my.second.gauge' assert second['points'][0][1] == 1.5 self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 1 assert second['metric'] == 'my.second.gauge' assert second['points'][0][1] == 1.5
def test_gauge_buckets(self): # Tests calling returing data from 2 time buckets ag_interval = self.interval stats = MetricsBucketAggregator('myhost', interval=ag_interval) self.wait_for_bucket_boundary(ag_interval) # Track some counters. stats.submit_packets('my.first.gauge:1|g') stats.submit_packets('my.first.gauge:5|g') stats.submit_packets('my.second.gauge:1.5|g') self.sleep_for_interval_length(ag_interval) stats.submit_packets('my.second.gauge:9.5|g') # Ensure that gauges roll up correctly. self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 3 first, second, second_b = metrics assert first['metric'] == 'my.first.gauge' assert first['points'][0][1] == 5 assert first['host'] == 'myhost' assert second_b['metric'] == 'my.second.gauge' assert second_b['points'][0][1] == 9.5 assert second['metric'] == 'my.second.gauge' assert second['points'][0][1] == 1.5 # check that they come back empty self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 0
def test_sampled_counter(self): # Submit a sampled counter. stats = MetricsBucketAggregator('myhost', interval=self.interval) stats.submit_packets('sampled.counter:1|c|@0.5') self.sleep_for_interval_length() metrics = stats.flush() assert len(metrics) == 1 m = metrics[0] assert m['metric'] == 'sampled.counter' assert m['points'][0][1] == 2
def test_histogram_flush_during_bucket(self): ag_interval = 1 # The min is not enabled by default stats = MetricsBucketAggregator( 'myhost', interval=ag_interval, histogram_aggregates=DEFAULT_HISTOGRAM_AGGREGATES + ['min']) # Sample all numbers between 1-100 many times. This # means our percentiles should be relatively close to themselves. self.wait_for_bucket_boundary(ag_interval) percentiles = list(range(100)) random.shuffle(percentiles) # in place for i in percentiles: for j in range(20): for type_ in ['h', 'ms']: m = 'my.p:%s|%s' % (i, type_) stats.submit_packets(m) time.sleep( self.BUCKET_BOUNDARY_TOLERANCE ) # Make sure that we'll wait for the _next_ bucket boundary self.wait_for_bucket_boundary(ag_interval) percentiles = list(range(50)) random.shuffle(percentiles) # in place for i in percentiles: for j in range(20): for type_ in ['h', 'ms']: m = 'my.p:%s|%s' % (i, type_) stats.submit_packets(m) metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 6 p95, pavg, pcount, pmax, pmed, pmin = self.sort_metrics(metrics) assert p95['metric'] == 'my.p.95percentile' self.assert_almost_equal(p95['points'][0][1], 95, 10) self.assert_almost_equal(pmax['points'][0][1], 99, 1) self.assert_almost_equal(pmed['points'][0][1], 50, 2) self.assert_almost_equal(pavg['points'][0][1], 50, 2) self.assert_almost_equal(pmin['points'][0][1], 1, 1) assert pcount['points'][0][1] == 4000 # 100 * 20 * 2 assert p95['host'] == 'myhost' self.sleep_for_interval_length() metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 6 p95_b, pavg_b, pcount_b, pmax_b, pmed_b, pmin_b = self.sort_metrics( metrics) assert p95_b['metric'] == 'my.p.95percentile' self.assert_almost_equal(p95_b['points'][0][1], 47, 10) self.assert_almost_equal(pmax_b['points'][0][1], 49, 1) self.assert_almost_equal(pmed_b['points'][0][1], 25, 2) self.assert_almost_equal(pavg_b['points'][0][1], 25, 2) self.assert_almost_equal(pmin_b['points'][0][1], 1, 1) assert pcount_b['points'][0][1] == 2000 # 100 * 20 * 2 # Ensure that histograms are reset. metrics = self.sort_metrics(stats.flush()) assert not metrics
def test_scientific_notation(self): ag_interval = 10 stats = MetricsBucketAggregator('myhost', interval=ag_interval) stats.submit_packets('test.scinot:9.512901e-05|g') self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 1 ts, val = metrics[0].get('points')[0] self.assert_almost_equal(val, 9.512901e-05)
def test_batch_submission(self): # Submit a sampled histogram. stats = MetricsBucketAggregator('myhost', interval=self.interval) metrics = ['counter:1|c', 'counter:1|c', 'gauge:1|g'] packet = "\n".join(metrics) stats.submit_packets(packet) self.sleep_for_interval_length() metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 2 counter, gauge = metrics assert counter['points'][0][1] == 2 assert gauge['points'][0][1] == 1
def test_gauge_sample_rate(self): ag_interval = self.interval stats = MetricsBucketAggregator('myhost', interval=ag_interval) # Submit a sampled gauge metric. stats.submit_packets('sampled.gauge:10|g|@0.1') # Assert that it's treated normally. self.sleep_for_interval_length(ag_interval) metrics = stats.flush() assert len(metrics) == 1 m = metrics[0] assert m['metric'] == 'sampled.gauge' assert m['points'][0][1] == 10
def init(config_path=None, use_watchdog=False, use_forwarder=False, args=None): """Configure the server and the reporting thread. """ c = get_config(parse_args=False, cfg_path=config_path) if not c['use_dogstatsd'] and \ (args and args[0] in ['start', 'restart'] or not args): log.info("Dogstatsd is disabled. Exiting") # We're exiting purposefully, so exit with zero (supervisor's expected # code). HACK: Sleep a little bit so supervisor thinks we've started cleanly # and thus can exit cleanly. sleep(4) sys.exit(0) log.debug("Configurating dogstatsd") port = c['dogstatsd_port'] interval = DOGSTATSD_FLUSH_INTERVAL api_key = c['api_key'] aggregator_interval = DOGSTATSD_AGGREGATOR_BUCKET_SIZE non_local_traffic = c['non_local_traffic'] forward_to_host = c.get('statsd_forward_host') forward_to_port = c.get('statsd_forward_port') event_chunk_size = c.get('event_chunk_size') recent_point_threshold = c.get('recent_point_threshold', None) target = c['dd_url'] if use_forwarder: target = c['dogstatsd_target'] hostname = get_hostname(c) # Create the aggregator (which is the point of communication between the # server and reporting threads. assert 0 < interval aggregator = MetricsBucketAggregator( hostname, aggregator_interval, recent_point_threshold=recent_point_threshold, formatter=get_formatter(c)) # Start the reporting thread. reporter = Reporter(interval, aggregator, target, api_key, use_watchdog, event_chunk_size) # Start the server on an IPv4 stack # Default to loopback server_host = c['bind_host'] # If specified, bind to all addressses if non_local_traffic: server_host = '' server = Server(aggregator, server_host, port, forward_to_host=forward_to_host, forward_to_port=forward_to_port) return reporter, server, c
def init5(agent_config=None, use_watchdog=False, use_forwarder=False, args=None): """Configure the server and the reporting thread. """ if (not agent_config['use_dogstatsd'] and (args and args[0] in ['start', 'restart'] or not args)): log.info("StsStatsd is disabled. Exiting") # We're exiting purposefully, so exit with zero (supervisor's expected # code). HACK: Sleep a little bit so supervisor thinks we've started cleanly # and thus can exit cleanly. sleep(4) sys.exit(0) port = agent_config['dogstatsd_port'] interval = DOGSTATSD_FLUSH_INTERVAL api_key = agent_config['api_key'] aggregator_interval = DOGSTATSD_AGGREGATOR_BUCKET_SIZE non_local_traffic = agent_config['non_local_traffic'] forward_to_host = agent_config.get('statsd_forward_host') forward_to_port = agent_config.get('statsd_forward_port') event_chunk_size = agent_config.get('event_chunk_size') recent_point_threshold = agent_config.get('recent_point_threshold', None) so_rcvbuf = agent_config.get('statsd_so_rcvbuf', None) server_host = agent_config['bind_host'] target = agent_config['dd_url'] if use_forwarder: target = agent_config['dogstatsd_target'] hostname = get_hostname(agent_config) log.debug("Using hostname \"%s\"", hostname) # Create the aggregator (which is the point of communication between the # server and reporting threads. assert 0 < interval aggregator = MetricsBucketAggregator( hostname, aggregator_interval, recent_point_threshold=recent_point_threshold, formatter=get_formatter(agent_config), histogram_aggregates=agent_config.get('histogram_aggregates'), histogram_percentiles=agent_config.get('histogram_percentiles'), utf8_decoding=agent_config['utf8_decoding'] ) # Start the reporting thread. reporter = Reporter(interval, aggregator, target, api_key, use_watchdog, event_chunk_size) # NOTICE: when `non_local_traffic` is passed we need to bind to any interface on the box. The forwarder uses # Tornado which takes care of sockets creation (more than one socket can be used at once depending on the # network settings), so it's enough to just pass an empty string '' to the library. # In Dogstatsd we use a single, fullstack socket, so passing '' as the address doesn't work and we default to # '0.0.0.0'. If someone needs to bind Dogstatsd to the IPv6 '::', they need to turn off `non_local_traffic` and # use the '::' meta address as `bind_host`. if non_local_traffic: server_host = '0.0.0.0' server = Server(aggregator, server_host, port, forward_to_host=forward_to_host, forward_to_port=forward_to_port, so_rcvbuf=so_rcvbuf) return reporter, server
def test_sampled_histogram(self): # Submit a sampled histogram. # The min is not enabled by default stats = MetricsBucketAggregator( 'myhost', interval=self.interval, histogram_aggregates=DEFAULT_HISTOGRAM_AGGREGATES + ['min']) stats.submit_packets('sampled.hist:5|h|@0.5') # Assert we scale up properly. self.sleep_for_interval_length() metrics = self.sort_metrics(stats.flush()) p95, pavg, pcount, pmax, pmed, pmin = self.sort_metrics(metrics) assert pcount['points'][0][1] == 2 for p in [p95, pavg, pmed, pmax, pmin]: assert p['points'][0][1] == 5
def init_dogstatsd(config): api_key = config['api_key'] recent_point_threshold = config.get('recent_point_threshold', None) server_host = config['dogstatsd']['bind_host'] dd_url = config['dd_url'] port = config['dogstatsd']['port'] forward_to_host = config['dogstatsd'].get('forward_host') forward_to_port = config['dogstatsd'].get('forward_port') non_local_traffic = config['dogstatsd'].get('non_local_traffic') so_rcvbuf = config['dogstatsd'].get('so_rcvbuf') utf8_decoding = config['dogstatsd'].get('utf8_decoding') interval = DOGSTATSD_FLUSH_INTERVAL aggregator_interval = DOGSTATSD_AGGREGATOR_BUCKET_SIZE hostname = get_hostname() # get proxy settings proxies = get_proxy() forwarder = Forwarder( api_key, dd_url, proxies=proxies, ) forwarder.start() aggregator = MetricsBucketAggregator( hostname, aggregator_interval, recent_point_threshold=recent_point_threshold, formatter=get_formatter(config), histogram_aggregates=config.get('histogram_aggregates'), histogram_percentiles=config.get('histogram_percentiles'), utf8_decoding=utf8_decoding ) # serializer serializer = Serializer( aggregator, forwarder, ) reporter = Reporter(interval, aggregator, serializer, api_key, use_watchdog=False, hostname=hostname) # NOTICE: when `non_local_traffic` is passed we need to bind to any interface on the box. The forwarder uses # Tornado which takes care of sockets creation (more than one socket can be used at once depending on the # network settings), so it's enough to just pass an empty string '' to the library. # In Dogstatsd we use a single, fullstack socket, so passing '' as the address doesn't work and we default to # '0.0.0.0'. If someone needs to bind Dogstatsd to the IPv6 '::', they need to turn off `non_local_traffic` and # use the '::' meta address as `bind_host`. if non_local_traffic: server_host = '0.0.0.0' server = Server(aggregator, server_host, port, forward_to_host=forward_to_host, forward_to_port=forward_to_port, so_rcvbuf=so_rcvbuf) return reporter, server, forwarder
def init(config_path=None, use_watchmonitor=False, use_forwarder=False, args=None): c = get_config(parse_args=False, cfg_path=config_path) if (not c['use_monitorstatsd'] and (args and args[0] in ['start', 'restart'] or not args)): log.info("Monitorstatsd is disabled. Exiting") sleep(4) sys.exit(0) log.debug("Configuring monitorstatsd") port = c['monitorstatsd_port'] interval = monitorSTATSD_FLUSH_INTERVAL api_key = c['api_key'] aggregator_interval = monitorSTATSD_AGGREGATOR_BUCKET_SIZE non_local_traffic = c['non_local_traffic'] forward_to_host = c.get('statsd_forward_host') forward_to_port = c.get('statsd_forward_port') event_chunk_size = c.get('event_chunk_size') recent_point_threshold = c.get('recent_point_threshold', None) ip = c.get('ip', "unknown") target = c['m_url'] if use_forwarder: target = c['monitorstatsd_target'] hostname = get_hostname(c) assert 0 < interval aggregator = MetricsBucketAggregator( hostname, aggregator_interval, recent_point_threshold=recent_point_threshold, formatter=get_formatter(c), histogram_aggregates=c.get('histogram_aggregates'), histogram_percentiles=c.get('histogram_percentiles'), utf8_decoding=c['utf8_decoding']) reporter = Reporter(c, interval, aggregator, target, api_key, use_watchmonitor, event_chunk_size) server_host = c['bind_host'] if non_local_traffic: server_host = '' server = Server(aggregator, server_host, port, forward_to_host=forward_to_host, forward_to_port=forward_to_port) return reporter, server, c
def test_sets(self): stats = MetricsBucketAggregator('myhost', interval=self.interval) stats.submit_packets('my.set:10|s') stats.submit_packets('my.set:20|s') stats.submit_packets('my.set:20|s') stats.submit_packets('my.set:30|s') stats.submit_packets('my.set:30|s') stats.submit_packets('my.set:30|s') # Assert that it's treated normally. self.sleep_for_interval_length() metrics = stats.flush() assert len(metrics) == 1 m = metrics[0] assert m['metric'] == 'my.set' assert m['points'][0][1] == 3 # Assert there are no more sets assert not stats.flush()
def test_calculate_bucket_start(self): stats = MetricsBucketAggregator('myhost', interval=10) assert stats.calculate_bucket_start(13284283) == 13284280 assert stats.calculate_bucket_start(13284280) == 13284280 stats = MetricsBucketAggregator('myhost', interval=5) assert stats.calculate_bucket_start(13284287) == 13284285 assert stats.calculate_bucket_start(13284280) == 13284280
def test_diagnostic_stats(self): stats = MetricsBucketAggregator('myhost', interval=self.interval) for i in range(10): stats.submit_packets('metric:10|c') stats.send_packet_count('datadog.dogstatsd.packet.count') self.sleep_for_interval_length() metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 2 first, second = metrics assert first['metric'] == 'datadog.dogstatsd.packet.count' assert first['points'][0][1] == 10
def test_dogstatsd_utf8_events(self): ma = MetricsBucketAggregator('my.host') for _ in xrange(self.FLUSH_COUNT): for i in xrange(self.LOOPS_PER_FLUSH): for j in xrange(self.METRIC_COUNT): ma.submit_packets( self.create_event_packet( 'Τη γλώσσα μου έδωσαν ελληνική', """τὸ σπίτι φτωχικὸ στὶς ἀμμουδιὲς τοῦ Ὁμήρου. Μονάχη ἔγνοια ἡ γλῶσσα μου στὶς ἀμμουδιὲς τοῦ Ὁμήρου. ἀπὸ τὸ Ἄξιον ἐστί τοῦ Ὀδυσσέα Ἐλύτη""" )) ma.submit_packets( self.create_event_packet( 'ვეპხის ტყაოსანი შოთა რუსთაველი', """ღმერთსი შემვედრე, ნუთუ კვლა დამხსნას სოფლისა შრომასა, ცეცხლს, წყალსა და მიწასა, ჰაერთა თანა მრომასა; მომცნეს ფრთენი და აღვფრინდე, მივჰხვდე მას ჩემსა ნდომასა, დღისით და ღამით ვჰხედვიდე მზისა ელვათა კრთომაასა. """)) ma.submit_packets( self.create_event_packet( 'Traité sur la tolérance', """Ose supposer qu'un Ministre éclairé & magnanime, un Prélat humain & sage, un Prince qui sait que son intérêt consiste dans le grand nombre de ses Sujets, & sa gloire dans leur bonheur, daigne jetter les yeux sur cet Ecrit informe & défectueux; il y supplée par ses propres lumieres; il se dit à lui-même: Que risquerai-je à voir la terre cultivée & ornée par plus de mains laborieuses, les tributs augmentés, l'Etat plus florissant?""" )) ma.flush()
def test_dogstatsd_ascii_events(self): ma = MetricsBucketAggregator('my.host') for _ in xrange(self.FLUSH_COUNT): for i in xrange(self.LOOPS_PER_FLUSH): for j in xrange(self.METRIC_COUNT): ma.submit_packets( self.create_event_packet( 'asldkfj fdsaljfas dflksjafs fasdfkjaldsfkjasldf', """alkdjfa slfalskdfjas lkfdjaoisudhfalsdkjbfaksdhfbasjdk fa;sf ljda fsafksadfh alsdjfhaskjdfgahls d;fjasdlkfh9823udjs dlfhaspdf98as ufdaksjhfaisdhufalskdjfhas df""" )) ma.submit_packets( self.create_event_packet( 'kdjfsofuousodifu982309rijdfsljsd dfsdf sdf', """dflskjdfs8d9fsdfjs sldfjka ;dlfjapfoia jsdflakjsdfp 0adsfuolwejf wflsdjf lsdkjf0saoiufja dlfjasd of;lasdjf ;askdjf asodfhas lkmfbashudf asd,fasdfna s,dfjas lcjx vjaskdlfjals dfkjasdflk jasldfkj asldkfjas ldfkasjdf a""" )) ma.submit_packets( self.create_event_packet( 'asdf askdjf asldfkjsad lfkajsdlfksajd fasdfsdfdf', """skdfjsld flskdjf alksdjfpasdofuapo sdfjalksdjf ;as.kjdf ;ljLKJL :KJL:KJ l;kdsjf ;lkj :Lkj FLDKFJ LSKFDJ ;LDFJ SLKDJF KSDLjf: Lfjldkj fLKDSJf lSKDjf ls;kdjf s;lkfjs L:KAJ :LFKJDL:DKjf L:SKjf;lKDJfl;SKJDf :LKSDj;lsdfj fsdljfsd ofisunafoialjsflmsdifjas;dlkfaj sdfkasjd flaksjdfnpmsao;difjkas dfnlaksdfa;sodljfas lfdjasdflmajsdlfknaf98wouanepr9qo3ud fadspuf oaisdufpoasid fj askdjn LKJH LKJHFL KJDHSF DSFLHSL JKDFHLSK DJFHLS KJDFHS""" )) ma.flush()
def test_bad_packets_throw_errors(self): packets = [ 'missing.value.and.type', 'missing.type:2', 'missing.value|c', '2|c', 'unknown.type:2|z', 'string.value:abc|c', 'string.sample.rate:0|c|@abc', # Bad event-like packets '_ev{1,2}:bad_header' '_e{1,}:invalid|headers', '_e:missing|size|headers', '_e:{1,1}:t|t|t:bad_meta|h', ] stats = MetricsBucketAggregator('myhost', interval=self.interval) for packet in packets: try: stats.submit_packets(packet) except Exception: assert True else: assert False, 'invalid : %s' % packet
def test_counter(self): ag_interval = 1.0 stats = MetricsBucketAggregator('myhost', interval=ag_interval) # Track some counters. stats.submit_packets('my.first.counter:1|c') stats.submit_packets('my.first.counter:5|c') stats.submit_packets('my.second.counter:1|c') stats.submit_packets('my.third.counter:3|c') # Ensure they roll up nicely. self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 3 first, second, third = metrics assert first['metric'] == 'my.first.counter' assert first['points'][0][1] == 6 assert first['host'] == 'myhost' assert second['metric'] == 'my.second.counter' assert second['points'][0][1] == 1 assert third['metric'] == 'my.third.counter' assert third['points'][0][1] == 3 self.sleep_for_interval_length(ag_interval) # Ensure that counters reset to zero. metrics = self.sort_metrics(stats.flush()) first, second, third = metrics assert first['metric'] == 'my.first.counter' assert first['points'][0][1] == 0 assert second['metric'] == 'my.second.counter' assert second['points'][0][1] == 0 assert third['metric'] == 'my.third.counter' assert third['points'][0][1] == 0
def test_sdstatsd_ascii_events(self): ma = MetricsBucketAggregator('my.host') for _ in xrange(self.FLUSH_COUNT): for i in xrange(self.LOOPS_PER_FLUSH): for j in xrange(self.METRIC_COUNT): ma.submit_packets(self.create_event_packet( 'asldkfj fdsaljfas dflksjafs fasdfkjaldsfkjasldf', """alkdjfa slfalskdfjas lkfdjaoisudhfalsdkjbfaksdhfbasjdk fa;sf ljda fsafksadfh alsdjfhaskjdfgahls d;fjasdlkfh9823udjs dlfhaspdf98as ufdaksjhfaisdhufalskdjfhas df""" )) ma.submit_packets(self.create_event_packet( 'kdjfsofuousodifu982309rijdfsljsd dfsdf sdf', """dflskjdfs8d9fsdfjs sldfjka ;dlfjapfoia jsdflakjsdfp 0adsfuolwejf wflsdjf lsdkjf0saoiufja dlfjasd of;lasdjf ;askdjf asodfhas lkmfbashudf asd,fasdfna s,dfjas lcjx vjaskdlfjals dfkjasdflk jasldfkj asldkfjas ldfkasjdf a""" )) ma.submit_packets(self.create_event_packet( 'asdf askdjf asldfkjsad lfkajsdlfksajd fasdfsdfdf', """skdfjsld flskdjf alksdjfpasdofuapo sdfjalksdjf ;as.kjdf ;ljLKJL :KJL:KJ l;kdsjf ;lkj :Lkj FLDKFJ LSKFDJ ;LDFJ SLKDJF KSDLjf: Lfjldkj fLKDSJf lSKDjf ls;kdjf s;lkfjs L:KAJ :LFKJDL:DKjf L:SKjf;lKDJfl;SKJDf :LKSDj;lsdfj fsdljfsd ofisunafoialjsflmsdifjas;dlkfaj sdfkasjd flaksjdfnpmsao;difjkas dfnlaksdfa;sodljfas lfdjasdflmajsdlfknaf98wouanepr9qo3ud fadspuf oaisdufpoasid fj askdjn LKJH LKJHFL KJDHSF DSFLHSL JKDFHLSK DJFHLS KJDFHS""" )) ma.flush()
def init(config_path=None, use_watchdog=False, use_forwarder=False): """Configure the server and the reporting thread. """ c = get_config(parse_args=False, cfg_path=config_path) log.debug("Configuration dogstatsd") port = c['dogstatsd_port'] interval = int(c['dogstatsd_interval']) aggregator_interval = int(c['dogstatsd_agregator_bucket_size']) api_key = c['api_key'] non_local_traffic = c['non_local_traffic'] forward_to_host = c.get('statsd_forward_host') forward_to_port = c.get('statsd_forward_port') event_chunk_size = c.get('event_chunk_size') target = c['dd_url'] if use_forwarder: target = c['dogstatsd_target'] hostname = get_hostname(c) # Create the aggregator (which is the point of communication between the # server and reporting threads. assert 0 < interval aggregator = MetricsBucketAggregator(hostname, aggregator_interval, recent_point_threshold=c.get( 'recent_point_threshold', None)) # Start the reporting thread. reporter = Reporter(interval, aggregator, target, api_key, use_watchdog, event_chunk_size) # Start the server on an IPv4 stack # Default to loopback server_host = c['bind_host'] # If specified, bind to all addressses if non_local_traffic: server_host = '' server = Server(aggregator, server_host, port, forward_to_host=forward_to_host, forward_to_port=forward_to_port) return reporter, server, c
def test_sdstatsd_utf8_events(self): ma = MetricsBucketAggregator('my.host') for _ in xrange(self.FLUSH_COUNT): for i in xrange(self.LOOPS_PER_FLUSH): for j in xrange(self.METRIC_COUNT): ma.submit_packets(self.create_event_packet( 'Τη γλώσσα μου έδωσαν ελληνική', """τὸ σπίτι φτωχικὸ στὶς ἀμμουδιὲς τοῦ Ὁμήρου. Μονάχη ἔγνοια ἡ γλῶσσα μου στὶς ἀμμουδιὲς τοῦ Ὁμήρου. ἀπὸ τὸ Ἄξιον ἐστί τοῦ Ὀδυσσέα Ἐλύτη""" )) ma.submit_packets(self.create_event_packet( 'ვეპხის ტყაოსანი შოთა რუსთაველი', """ღმერთსი შემვედრე, ნუთუ კვლა დამხსნას სოფლისა შრომასა, ცეცხლს, წყალსა და მიწასა, ჰაერთა თანა მრომასა; მომცნეს ფრთენი და აღვფრინდე, მივჰხვდე მას ჩემსა ნდომასა, დღისით და ღამით ვჰხედვიდე მზისა ელვათა კრთომაასა. """ )) ma.submit_packets(self.create_event_packet( 'Traité sur la tolérance', """Ose supposer qu'un Ministre éclairé & magnanime, un Prélat humain & sage, un Prince qui sait que son intérêt consiste dans le grand nombre de ses Sujets, & sa gloire dans leur bonheur, daigne jetter les yeux sur cet Ecrit informe & défectueux; il y supplée par ses propres lumieres; il se dit à lui-même: Que risquerai-je à voir la terre cultivée & ornée par plus de mains laborieuses, les tributs augmentés, l'Etat plus florissant?""" )) ma.flush()
def test_counter_normalization(self): ag_interval = 10 stats = MetricsBucketAggregator('myhost', interval=ag_interval) # Assert counters are normalized. stats.submit_packets('int:1|c') stats.submit_packets('int:4|c') stats.submit_packets('int:15|c') stats.submit_packets('float:5|c') self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 2 floatc, intc = metrics assert floatc['metric'] == 'float' assert floatc['points'][0][1] == 0.5 assert floatc['host'] == 'myhost' assert intc['metric'] == 'int' assert intc['points'][0][1] == 2 assert intc['host'] == 'myhost'
def test_histogram_normalization(self): ag_interval = 10 # The min is not enabled by default stats = MetricsBucketAggregator( 'myhost', interval=ag_interval, histogram_aggregates=DEFAULT_HISTOGRAM_AGGREGATES + ['min']) for i in range(5): stats.submit_packets('h1:1|h') for i in range(20): stats.submit_packets('h2:1|h') self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) _, _, h1count, _, _, _, _, _, h2count, _, _, _ = metrics assert h1count['points'][0][1] == 0.5 assert h2count['points'][0][1] == 2
def test_empty_counter(self): ag_interval = self.interval stats = MetricsBucketAggregator('myhost', interval=ag_interval) self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) # Should be an empty list assert len(metrics) == 0 # Track some counters. stats.submit_packets('my.first.counter:%s|c' % (1 * ag_interval)) # Call flush before the bucket_length has been exceeded metrics = self.sort_metrics(stats.flush()) # Should be an empty list assert len(metrics) == 0 self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) # Should now have the data assert len(metrics) == 1 assert metrics[0]['metric'] == 'my.first.counter' assert metrics[0]['points'][0][1] == 1
def init(server_host, port, timeout=UDP_SOCKET_TIMEOUT, aggregator_interval=DOGSTATSD_AGGREGATOR_BUCKET_SIZE): """Configure the server and the reporting thread. """ log.debug("Configuring dogstatsd") hostname = None aggregator = MetricsBucketAggregator( hostname, aggregator_interval, recent_point_threshold=None, formatter=None, histogram_aggregates=DEFAULT_HISTOGRAM_AGGREGATES, histogram_percentiles=DEFAULT_HISTOGRAM_PERCENTILES, utf8_decoding=True, ) server = Server(aggregator, server_host, port, timeout=timeout) return server
def test_histogram(self): ag_interval = self.interval # The min is not enabled by default stats = MetricsBucketAggregator( 'myhost', interval=ag_interval, histogram_aggregates=DEFAULT_HISTOGRAM_AGGREGATES + ['min']) self.wait_for_bucket_boundary(ag_interval) # Sample all numbers between 1-100 many times. This # means our percentiles should be relatively close to themselves. percentiles = range(100) random.shuffle(percentiles) # in place for i in percentiles: for j in xrange(20): for type_ in ['h', 'ms']: m = 'my.p:%s|%s' % (i, type_) stats.submit_packets(m) self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 6 p95, pavg, pcount, pmax, pmed, pmin = self.sort_metrics(metrics) assert p95['metric'] == 'my.p.95percentile' self.assert_almost_equal(p95['points'][0][1], 95, 10) self.assert_almost_equal(pmax['points'][0][1], 99, 1) self.assert_almost_equal(pmed['points'][0][1], 50, 2) self.assert_almost_equal(pavg['points'][0][1], 50, 2) self.assert_almost_equal(pmin['points'][0][1], 1, 1) assert pcount['points'][0][1] == 4000 # 100 * 20 * 2 assert p95['host'] == 'myhost' # Ensure that histograms are reset. metrics = self.sort_metrics(stats.flush()) assert not metrics
def test_event_title(self): stats = MetricsBucketAggregator('myhost', interval=self.interval) stats.submit_packets('_e{0,4}:|text') stats.submit_packets('_e{9,4}:2intitulé|text') stats.submit_packets('_e{14,4}:3title content|text') stats.submit_packets('_e{14,4}:4title|content|text') stats.submit_packets( '_e{13,4}:5title\\ntitle|text') # \n stays escaped events = self.sort_events(stats.flush_events()) assert len(events) == 5 first, second, third, fourth, fifth = events assert first['msg_title'] == '' assert second['msg_title'] == '2intitulé' assert third['msg_title'] == '3title content' assert fourth['msg_title'] == '4title|content' assert fifth['msg_title'] == '5title\\ntitle'
def test_dogstatsd_aggregation_perf(self): ma = MetricsBucketAggregator('my.host') for _ in xrange(self.FLUSH_COUNT): for i in xrange(self.LOOPS_PER_FLUSH): for j in xrange(self.METRIC_COUNT): # metrics ma.submit_packets('counter.%s:%s|c' % (j, i)) ma.submit_packets('gauge.%s:%s|g' % (j, i)) ma.submit_packets('histogram.%s:%s|h' % (j, i)) ma.submit_packets('set.%s:%s|s' % (j, 1.0)) # tagged metrics ma.submit_packets('counter.%s:%s|c|#tag1,tag2' % (j, i)) ma.submit_packets('gauge.%s:%s|g|#tag1,tag2' % (j, i)) ma.submit_packets('histogram.%s:%s|h|#tag1,tag2' % (j, i)) ma.submit_packets('set.%s:%s|s|#tag1,tag2' % (j, i)) # sampled metrics ma.submit_packets('counter.%s:%s|c|@0.5' % (j, i)) ma.submit_packets('gauge.%s:%s|g|@0.5' % (j, i)) ma.submit_packets('histogram.%s:%s|h|@0.5' % (j, i)) ma.submit_packets('set.%s:%s|s|@0.5' % (j, i)) ma.flush()
def test_event_text(self): stats = MetricsBucketAggregator('myhost', interval=self.interval) stats.submit_packets('_e{2,0}:t1|') stats.submit_packets('_e{2,12}:t2|text|content') stats.submit_packets( '_e{2,23}:t3|First line\\nSecond line') # \n is a newline stats.submit_packets( '_e{2,19}:t4|♬ †øU †øU ¥ºu T0µ ♪') # utf-8 compliant events = self.sort_events(stats.flush_events()) assert len(events) == 4 first, second, third, fourth = events assert first['msg_text'] == '' assert second['msg_text'] == 'text|content' assert third['msg_text'] == 'First line\nSecond line' assert fourth['msg_text'] == '♬ †øU †øU ¥ºu T0µ ♪'
def test_recent_point_threshold(self): ag_interval = 1 threshold = 100 # The min is not enabled by default stats = MetricsBucketAggregator( 'myhost', recent_point_threshold=threshold, interval=ag_interval, histogram_aggregates=DEFAULT_HISTOGRAM_AGGREGATES + ['min']) timestamp_beyond_threshold = time.time() - threshold * 2 # Ensure that old gauges get dropped due to old timestamps stats.submit_metric('my.first.gauge', 5, 'g') stats.submit_metric('my.first.gauge', 1, 'g', timestamp=timestamp_beyond_threshold) stats.submit_metric('my.second.gauge', 20, 'g', timestamp=timestamp_beyond_threshold) self.sleep_for_interval_length(ag_interval) metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 1 first = metrics[0] assert first['metric'] == 'my.first.gauge' assert first['points'][0][1] == 5 assert first['host'] == 'myhost' timestamp_within_threshold = time.time() - threshold / 2 bucket_for_timestamp_within_threshold = timestamp_within_threshold - ( timestamp_within_threshold % ag_interval) stats.submit_metric('my.1.gauge', 5, 'g') stats.submit_metric('my.1.gauge', 1, 'g', timestamp=timestamp_within_threshold) stats.submit_metric('my.2.counter', 20, 'c', timestamp=timestamp_within_threshold) stats.submit_metric('my.3.set', 20, 's', timestamp=timestamp_within_threshold) stats.submit_metric('my.4.histogram', 20, 'h', timestamp=timestamp_within_threshold) self.sleep_for_interval_length(ag_interval) flush_timestamp = time.time() # The bucket timestamp is the beginning of the bucket that ended before we flushed bucket_timestamp = flush_timestamp - (flush_timestamp % ag_interval) - ag_interval metrics = self.sort_metrics(stats.flush()) assert len(metrics) == 11 first, first_b, second, second_b, third, h1, h2, h3, h4, h5, h6 = metrics assert first['metric'] == 'my.1.gauge' assert first['points'][0][1] == 1 assert first['host'] == 'myhost' self.assert_almost_equal(first['points'][0][0], bucket_for_timestamp_within_threshold, 0.1) assert first_b['metric'] == 'my.1.gauge' assert first_b['points'][0][1] == 5 self.assert_almost_equal(first_b['points'][0][0], bucket_timestamp, 0.1) assert second['metric'] == 'my.2.counter' assert second['points'][0][1] == 20 self.assert_almost_equal(second['points'][0][0], bucket_for_timestamp_within_threshold, 0.1) assert second_b['metric'] == 'my.2.counter' assert second_b['points'][0][1] == 0 self.assert_almost_equal(second_b['points'][0][0], bucket_timestamp, 0.1) assert third['metric'] == 'my.3.set' assert third['points'][0][1] == 1 self.assert_almost_equal(third['points'][0][0], bucket_for_timestamp_within_threshold, 0.1) assert h1['metric'] == 'my.4.histogram.95percentile' assert h1['points'][0][1] == 20 self.assert_almost_equal(h1['points'][0][0], bucket_for_timestamp_within_threshold, 0.1) assert h1['points'][0][0] == h2['points'][0][0] assert h1['points'][0][0] == h3['points'][0][0] assert h1['points'][0][0] == h4['points'][0][0] assert h1['points'][0][0] == h5['points'][0][0]
def test_sdstatsd_aggregation_perf(self): ma = MetricsBucketAggregator('my.host') for _ in xrange(self.FLUSH_COUNT): for i in xrange(self.LOOPS_PER_FLUSH): for j in xrange(self.METRIC_COUNT): # metrics ma.submit_packets('counter.%s:%s|c' % (j, i)) ma.submit_packets('gauge.%s:%s|g' % (j, i)) ma.submit_packets('histogram.%s:%s|h' % (j, i)) ma.submit_packets('set.%s:%s|s' % (j, 1.0)) # tagged metrics ma.submit_packets('counter.%s:%s|c|#tag1,tag2' % (j, i)) ma.submit_packets('gauge.%s:%s|g|#tag1,tag2' % (j, i)) ma.submit_packets('histogram.%s:%s|h|#tag1,tag2' % (j, i)) ma.submit_packets('set.%s:%s|s|#tag1,tag2' % (j, i)) # sampled metrics ma.submit_packets('counter.%s:%s|c|@0.5' % (j, i)) ma.submit_packets('gauge.%s:%s|g|@0.5' % (j, i)) ma.submit_packets('histogram.%s:%s|h|@0.5' % (j, i)) ma.submit_packets('set.%s:%s|s|@0.5' % (j, i)) ma.flush()
def test_event_tags(self): stats = MetricsBucketAggregator('myhost', interval=self.interval) stats.submit_packets('_e{6,4}:title1|text') stats.submit_packets('_e{6,4}:title2|text|#t1') stats.submit_packets('_e{6,4}:title3|text|#t1,t2:v2,t3,t4') stats.submit_packets('_e{6,4}:title4|text|k:key|p:normal|#t1,t2') events = self.sort_events(stats.flush_events()) assert len(events) == 4 first, second, third, fourth = events try: first['tags'] except Exception: assert True else: assert False, "event['tags'] shouldn't be defined when no tags aren't explicited in the packet" assert first['msg_title'] == 'title1' assert first['msg_text'] == 'text' assert second['msg_title'] == 'title2' assert second['msg_text'] == 'text' assert second['tags'] == sorted(['t1']) assert third['msg_title'] == 'title3' assert third['msg_text'] == 'text' assert third['tags'] == sorted(['t1', 't2:v2', 't3', 't4']) assert fourth['msg_title'] == 'title4' assert fourth['msg_text'] == 'text' assert fourth['aggregation_key'] == 'key' assert fourth['priority'] == 'normal' assert fourth['tags'] == sorted(['t1', 't2'])