class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.path.dirname(__file__))
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def testTomcatMetrics(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 25:
raise Exception("No metrics were received in 25 seconds")
metrics = self.reporter.metrics
self.assertTrue(isinstance(metrics, ListType))
self.assertTrue(len(metrics) > 8, metrics)
self.assertEquals(len([t for t in metrics if 'instance:solr_instance' in t['tags'] and t['metric'] == "jvm.thread_count"]), 1, metrics)
self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and 'instance:solr_instance' in t['tags']]) > 4, metrics)
self.assertTrue(len([t for t in metrics if "solr." in t['metric'] and 'instance:solr_instance' in t['tags']]) > 4, metrics)
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls"))
JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
def tearDown(self):
self.server.stop()
self.reporter.finished = True
JMXFetch.stop()
def testTomcatMetrics(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 20:
raise Exception("No metrics were received in 20 seconds")
metrics = self.reporter.metrics
self.assertTrue(type(metrics) == type([]))
self.assertTrue(len(metrics) > 0)
self.assertEquals(len([t for t in metrics if t['metric'] == "tomcat.threads.busy" and "instance:tomcat_instance" in t['tags']]), 2, metrics)
self.assertEquals(len([t for t in metrics if t['metric'] == "tomcat.bytes_sent" and "instance:tomcat_instance" in t['tags']]), 0, metrics)
self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and "instance:tomcat_instance" in t['tags']]) > 4, metrics)
class TestTomcat(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml')
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def test_tomcat_metrics(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 25:
raise Exception("No metrics were received in 25 seconds")
metrics = self.reporter.metrics
self.assertTrue(type(metrics) == type([]))
self.assertTrue(len(metrics) > 0)
self.assertEquals(len([t for t in metrics if t['metric'] == "tomcat.threads.busy" and "instance:tomcat_instance" in t['tags']]), 2, metrics)
self.assertEquals(len([t for t in metrics if t['metric'] == "tomcat.bytes_sent" and "instance:tomcat_instance" in t['tags']]), 0, metrics)
self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and "instance:tomcat_instance" in t['tags']]) > 4, metrics)
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml')
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 20:
raise Exception("No metrics were received in 20 seconds")
metrics = self.reporter.metrics
self.assertTrue(isinstance(metrics, ListType))
self.assertTrue(len(metrics) > 0)
self.assertEquals(len([t for t in metrics if t['metric'] == "my.metric.buf" and "instance:jmx_instance1" in t['tags']]), 2, metrics)
self.assertTrue(len([t for t in metrics if 'type:ThreadPool' in t['tags'] and "instance:jmx_instance1" in t['tags'] and "jmx.catalina" in t['metric']]) > 8, metrics)
self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and "instance:jmx_instance1" in t['tags']]) == 13, metrics)
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls"))
JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15)
def tearDown(self):
self.server.stop()
self.reporter.finished = True
JMXFetch.stop()
def testTomcatMetrics(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 20:
raise Exception("No metrics were received in 20 seconds")
metrics = self.reporter.metrics
self.assertTrue(type(metrics) == type([]))
self.assertTrue(len(metrics) > 0)
self.assertEquals(len([t for t in metrics if t['metric'] == "tomcat.threads.busy" and "instance:tomcat_instance" in t['tags']]), 2, metrics)
self.assertEquals(len([t for t in metrics if t['metric'] == "tomcat.bytes_sent" and "instance:tomcat_instance" in t['tags']]), 0, metrics)
self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and "instance:tomcat_instance" in t['tags']]) > 4, metrics)
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.path.dirname(__file__))
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def testTomcatMetrics(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 25:
raise Exception("No metrics were received in 25 seconds")
metrics = self.reporter.metrics
self.assertTrue(isinstance(metrics, ListType))
self.assertTrue(len(metrics) > 8, metrics)
self.assertEquals(len([t for t in metrics if 'instance:solr_instance' in t['tags'] and t['metric'] == "jvm.thread_count"]), 1, metrics)
self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and 'instance:solr_instance' in t['tags']]) > 4, metrics)
self.assertTrue(len([t for t in metrics if "solr." in t['metric'] and 'instance:solr_instance' in t['tags']]) > 4, metrics)
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = Fixtures.directory()
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 25:
raise Exception("No metrics were received in 25 seconds")
metrics = self.reporter.metrics
self.assertTrue(isinstance(metrics, ListType))
self.assertTrue(len(metrics) > 0)
self.assertTrue(len([t for t in metrics if "cassandra.db." in t['metric'] and "instance:cassandra_instance" in t['tags']]) > 40, metrics)
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml')
JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
def tearDown(self):
self.server.stop()
self.reporter.finished = True
JMXFetch.stop()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 20:
raise Exception("No metrics were received in 20 seconds")
metrics = self.reporter.metrics
self.assertTrue(type(metrics) == type([]))
self.assertTrue(len(metrics) > 0)
self.assertEquals(len([t for t in metrics if t['metric'] == "my.metric.buf" and "instance:jmx_instance1" in t['tags']]), 2, metrics)
self.assertTrue(len([t for t in metrics if 'type:ThreadPool' in t['tags'] and "instance:jmx_instance1" in t['tags'] and "jmx.catalina" in t['metric']]) > 8, metrics)
self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and "instance:jmx_instance1" in t['tags']]) == 7, metrics)
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls"))
JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
def tearDown(self):
self.server.stop()
self.reporter.finished = True
JMXFetch.stop()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 25:
raise Exception("No metrics were received in 25 seconds")
metrics = self.reporter.metrics
self.assertTrue(type(metrics) == type([]))
self.assertTrue(len(metrics) > 0)
self.assertTrue(len([t for t in metrics if "cassandra.db." in t['metric'] and "instance:cassandra_instance" in t['tags']]) > 40, metrics)
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = Fixtures.directory()
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 25:
raise Exception("No metrics were received in 25 seconds")
metrics = self.reporter.metrics
self.assertTrue(isinstance(metrics, ListType))
self.assertTrue(len(metrics) > 0)
self.assertTrue(len([t for t in metrics if "cassandra.db." in t['metric'] and "instance:cassandra_instance" in t['tags']]) > 40, metrics)
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls"))
JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = Fixtures.directory()
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml')
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.path.dirname(__file__), 'ci/resources/')
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
class TestKafka(unittest.TestCase):
"""Basic Test for kafka integration."""
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.path.dirname(__file__), 'resources/')
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 25:
raise Exception("No metrics were received in 25 seconds")
metrics = self.reporter.metrics
# expected_tags = ['env:test', 'instance:kafka-172.17.0.1-9999', 'kafka:broker']
self.assertTrue(isinstance(metrics, ListType))
self.assertTrue(len(metrics) > 0)
log.info(metrics)
log.info(len(metrics))
self.assertTrue(
len([t for t in metrics if "jvm." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) >= 13, metrics)
self.assertTrue(
len([t for t in metrics if "kafka.request." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) == 12, metrics)
self.assertTrue(
len([t for t in metrics if "kafka.replication." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) == 6, metrics)
# CLIENT metrics.
# kafka.producer.request_latency_avg
self.assertTrue(
len([t for t in metrics if "kafka.producer." in t['metric'] and "instance:kafka-172.17.0.1-7777" in t['tags']]) == 1, metrics)
# kafka.consumer.fetch_rate, kafka.consumer.max_lag
self.assertTrue(
len([t for t in metrics if "kafka.consumer." in t['metric'] and "instance:kafka-172.17.0.1-7777" in t['tags']]) == 2, metrics)
class TestKafka(unittest.TestCase):
"""Basic Test for kafka integration."""
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.path.dirname(__file__), 'ci/resources/')
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 25:
raise Exception("No metrics were received in 25 seconds")
metrics = self.reporter.metrics
# expected_tags = ['env:test', 'instance:kafka-172.17.0.1-9999', 'kafka:broker']
self.assertTrue(isinstance(metrics, ListType))
self.assertTrue(len(metrics) > 0)
log.info(metrics)
log.info(len(metrics))
self.assertTrue(
len([t for t in metrics if "jvm." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) >= 13, metrics)
self.assertTrue(
len([t for t in metrics if "kafka.request." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) == 12, metrics)
self.assertTrue(
len([t for t in metrics if "kafka.replication." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) == 6, metrics)
# CLIENT metrics.
# kafka.producer.request_latency_avg
self.assertTrue(
len([t for t in metrics if "kafka.producer." in t['metric'] and "instance:kafka-172.17.0.1-7777" in t['tags']]) == 1, metrics)
# kafka.consumer.fetch_rate, kafka.consumer.max_lag
self.assertTrue(
len([t for t in metrics if "kafka.consumer." in t['metric'] and "instance:kafka-172.17.0.1-7777" in t['tags']]) == 2, metrics)
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.realpath(
os.path.join(os.path.abspath(__file__), "..", "jmx_yamls"))
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml')
JMXFetch.init(confd_path, {'dogstatsd_port': STATSD_PORT},
get_logging_config(), 15, JMX_COLLECT_COMMAND)
def tearDown(self):
self.server.stop()
self.reporter.finished = True
JMXFetch.stop()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 20:
raise Exception("No metrics were received in 20 seconds")
metrics = self.reporter.metrics
self.assertTrue(type(metrics) == type([]))
self.assertTrue(len(metrics) > 0)
self.assertEquals(
len([
t for t in metrics if t['metric'] == "my.metric.buf"
and "instance:jmx_instance1" in t['tags']
]), 2, metrics)
self.assertTrue(
len([
t for t in metrics
if 'type:ThreadPool' in t['tags'] and "instance:jmx_instance1"
in t['tags'] and "jmx.catalina" in t['metric']
]) > 8, metrics)
self.assertTrue(
len([
t for t in metrics if "jvm." in t['metric']
and "instance:jmx_instance1" in t['tags']
]) == 7, metrics)
def test_start(self, select):
select.select.side_effect = [KeyboardInterrupt, SystemExit]
s1 = Server(mock.MagicMock(), '::1', '1234')
s1.start()
self.assertEqual(s1.socket.family, socket.AF_INET6)
s2 = Server(mock.MagicMock(), '127.0.0.1', '2345')
s2.start()
self.assertEqual(s2.socket.family, socket.AF_INET6)
s2 = Server(mock.MagicMock(), 'foo', '80')
s2.start()
self.assertFalse(s2.running)
def test_init(self, nh):
nh.return_value = 'foo'
s = Server(None, 'localhost', '1234')
nh.assertCalledOnceWith('localhost', 1234)
self.assertEqual(s.sockaddr, 'foo')
self.assertIsNone(s.socket)
class TestHbase_master(unittest.TestCase):
"""Basic Test for hbase_master integration."""
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.path.dirname(__file__), 'ci/resources/')
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 60:
raise Exception("No metrics were received in 60 seconds")
metrics = self.reporter.metrics
self.assertTrue(isinstance(metrics, ListType))
self.assertTrue(len(metrics) > 0)
self.assertTrue(
len([
t for t in metrics if "jvm." in t['metric']
and "instance:hbase_master-localhost-10101" in t['tags']
]) >= 13, metrics)
# 'Hadoop:service=HBase,name=Master,sub=Server - tag.isActiveMaster' needs to be converted to number. But 'values:'(valueConversion) doesn't work in yaml...
self.assertTrue(
len([
t for t in self.reporter.metrics
if "hbase.master." in t['metric']
and "instance:hbase_master-localhost-10101" in t['tags']
]) >= 20, metrics)
def init_dogstatsd(config):
api_key = config['api_key']
recent_point_threshold = config.get('recent_point_threshold', None)
server_host = config['dogstatsd']['bind_host']
dd_url = config['dd_url']
port = config['dogstatsd']['port']
forward_to_host = config['dogstatsd'].get('forward_host')
forward_to_port = config['dogstatsd'].get('forward_port')
non_local_traffic = config['dogstatsd'].get('non_local_traffic')
so_rcvbuf = config['dogstatsd'].get('so_rcvbuf')
utf8_decoding = config['dogstatsd'].get('utf8_decoding')
interval = DOGSTATSD_FLUSH_INTERVAL
aggregator_interval = DOGSTATSD_AGGREGATOR_BUCKET_SIZE
hostname = get_hostname()
# get proxy settings
proxies = get_proxy()
forwarder = Forwarder(
api_key,
dd_url,
proxies=proxies,
)
forwarder.start()
aggregator = MetricsBucketAggregator(
hostname,
aggregator_interval,
recent_point_threshold=recent_point_threshold,
formatter=get_formatter(config),
histogram_aggregates=config.get('histogram_aggregates'),
histogram_percentiles=config.get('histogram_percentiles'),
utf8_decoding=utf8_decoding
)
# serializer
serializer = Serializer(
aggregator,
forwarder,
)
reporter = Reporter(interval, aggregator, serializer, api_key,
use_watchdog=False, hostname=hostname)
# NOTICE: when `non_local_traffic` is passed we need to bind to any interface on the box. The forwarder uses
# Tornado which takes care of sockets creation (more than one socket can be used at once depending on the
# network settings), so it's enough to just pass an empty string '' to the library.
# In Dogstatsd we use a single, fullstack socket, so passing '' as the address doesn't work and we default to
# '0.0.0.0'. If someone needs to bind Dogstatsd to the IPv6 '::', they need to turn off `non_local_traffic` and
# use the '::' meta address as `bind_host`.
if non_local_traffic:
server_host = '0.0.0.0'
server = Server(aggregator, server_host, port, forward_to_host=forward_to_host,
forward_to_port=forward_to_port, so_rcvbuf=so_rcvbuf)
return reporter, server, forwarder
def test_start(self, select):
select.select.side_effect = [KeyboardInterrupt, SystemExit]
s1 = Server(mock.MagicMock(), '::1', '1234')
s1.start()
self.assertEqual(s1.socket.family, socket.AF_INET6)
original_so_rcvbuf = s1.socket.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)
s2 = Server(mock.MagicMock(), '127.0.0.1', '2345', so_rcvbuf=1023)
s2.start()
self.assertEqual(s2.socket.family, socket.AF_INET6)
self.assertNotEquals(original_so_rcvbuf, s2.socket.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
s2 = Server(mock.MagicMock(), 'foo', '80')
s2.start()
self.assertFalse(s2.running)
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls"))
JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15)
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml')
JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
class JMXTestCase(unittest.TestCase):
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile("dogstatsd")
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls"))
JMXFetch.init(confd_path, {"dogstatsd_port": STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
def tearDown(self):
self.server.stop()
self.reporter.finished = True
JMXFetch.stop()
def testTomcatMetrics(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 25:
raise Exception("No metrics were received in 25 seconds")
metrics = self.reporter.metrics
self.assertTrue(type(metrics) == type([]))
self.assertTrue(len(metrics) > 8, metrics)
self.assertEquals(
len([t for t in metrics if "instance:solr_instance" in t["tags"] and t["metric"] == "jvm.thread_count"]),
1,
metrics,
)
self.assertTrue(
len([t for t in metrics if "jvm." in t["metric"] and "instance:solr_instance" in t["tags"]]) > 4, metrics
)
self.assertTrue(
len([t for t in metrics if "solr." in t["metric"] and "instance:solr_instance" in t["tags"]]) > 4, metrics
)
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.path.dirname(__file__))
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml')
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
pid_file = PidFile('dogstatsd')
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = Fixtures.directory()
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
class TestHbase_regionserver(unittest.TestCase):
"""Basic Test for hbase_regionserver integration."""
def setUp(self):
aggregator = MetricsAggregator("test_host")
self.server = Server(aggregator, "localhost", STATSD_PORT)
self.reporter = DummyReporter(aggregator)
self.t1 = threading.Thread(target=self.server.start)
self.t1.start()
confd_path = os.path.join(os.path.dirname(__file__), 'ci/resources/')
self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT})
self.t2 = threading.Thread(target=self.jmx_daemon.run)
self.t2.start()
def tearDown(self):
self.server.stop()
self.reporter.finished = True
self.jmx_daemon.terminate()
def testCustomJMXMetric(self):
count = 0
while self.reporter.metrics is None:
time.sleep(1)
count += 1
if count > 60:
raise Exception("No metrics were received in 60 seconds")
metrics = self.reporter.metrics
self.assertTrue(isinstance(metrics, list))
self.assertTrue(len(metrics) > 0)
self.assertTrue(
len([
t for t in metrics if "jvm." in t['metric']
and "instance:hbase_regionserver-localhost-10102" in t['tags']
]) >= 13, metrics)
# waiting for receiving metrics which appears after a while.
count = 0
while True:
metrics = self.reporter.metrics
mutations_metrics = [
t for t in metrics
if "hbase.regionserver.server.mutations" in t['metric']
and "instance:hbase_regionserver-localhost-10102" in t['tags']
]
slow_appned_metrics = [
t for t in metrics
if "hbase.regionserver.server.slow_append" in t['metric']
and "instance:hbase_regionserver-localhost-10102" in t['tags']
]
# hedged_metrics = [t for t in metrics if "hbase.regionserver.server.hedged_read" in t['metric'] and "instance:hbase_regionserver-localhost-10102" in t['tags']]
# pause_time_metrics = [t for t in metrics if "hbase.regionserver.server.pause_time" in t['metric'] and "instance:hbase_regionserver-localhost-10102" in t['tags']]
time.sleep(1)
count += 1
if len(mutations_metrics) >= 2 and len(slow_appned_metrics) >= 1:
break
elif count <= 60:
continue
else:
log.info(metrics)
raise Exception("enough metrics were received in 60 seconds")
# hbase.regionserver.server.hedged_read* and hbase.regionserver.server.pause* is ignored here
# because these metrics won't emit until these process will actually happen.
metrics = [
t for t in self.reporter.metrics if "hbase." in t['metric']
and "instance:hbase_regionserver-localhost-10102" in t['tags']
]
num_total = 159
num_hedged_read = 2
num_pause = 14
num = len(metrics)
log.info(num)
self.assertTrue(num >= (num_total - num_hedged_read - num_pause),
metrics)
def test_start(self, select):
select.select.side_effect = [KeyboardInterrupt, SystemExit]
s1 = Server(mock.MagicMock(), '::1', '1234')
s1.start()
self.assertEqual(s1.socket.family, socket.AF_INET6)
original_so_rcvbuf = s1.socket.getsockopt(socket.SOL_SOCKET,
socket.SO_RCVBUF)
s2 = Server(mock.MagicMock(), '127.0.0.1', '2345', so_rcvbuf=1023)
s2.start()
self.assertEqual(s2.socket.family, socket.AF_INET6)
self.assertNotEquals(
original_so_rcvbuf,
s2.socket.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
s2 = Server(mock.MagicMock(), 'foo', '80')
s2.start()
self.assertFalse(s2.running)
def test_init(self):
s = Server(None, 'localhost', '1234')
self.assertIsNone(s.sockaddr)
self.assertIsNone(s.socket)
def test_start(self, select):
select.select.side_effect = [KeyboardInterrupt, SystemExit]
s1 = Server(mock.MagicMock(), '::1', '1234')
s1.start()
self.assertEqual(s1.socket.family, socket.AF_INET6)
s2 = Server(mock.MagicMock(), '127.0.0.1', '2345')
s2.start()
self.assertEqual(s2.socket.family, socket.AF_INET6)
s2 = Server(mock.MagicMock(), 'foo', '80')
s2.start()
self.assertFalse(s2.running)
