class JMXTestCase(unittest.TestCase): def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml') self.jmx_daemon = JMXFetch(confd_path, {'sdstatsd_port': STATSD_PORT}) self.t2 = threading.Thread(target=self.jmx_daemon.run) self.t2.start() def tearDown(self): self.server.stop() self.reporter.finished = True self.jmx_daemon.terminate() def testCustomJMXMetric(self): count = 0 while self.reporter.metrics is None: time.sleep(1) count += 1 if count > 20: raise Exception("No metrics were received in 20 seconds") metrics = self.reporter.metrics self.assertTrue(isinstance(metrics, ListType)) self.assertTrue(len(metrics) > 0) self.assertEquals(len([t for t in metrics if t['metric'] == "my.metric.buf" and "instance:jmx_instance1" in t['tags']]), 2, metrics) self.assertTrue(len([t for t in metrics if 'type:ThreadPool' in t['tags'] and "instance:jmx_instance1" in t['tags'] and "jmx.catalina" in t['metric']]) > 8, metrics) self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and "instance:jmx_instance1" in t['tags']]) == 13, metrics)
class TestTomcat(unittest.TestCase): def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) pid_file = PidFile('dogstatsd') self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml') self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT}) self.t2 = threading.Thread(target=self.jmx_daemon.run) self.t2.start() def tearDown(self): self.server.stop() self.reporter.finished = True self.jmx_daemon.terminate() def test_tomcat_metrics(self): count = 0 while self.reporter.metrics is None: time.sleep(1) count += 1 if count > 25: raise Exception("No metrics were received in 25 seconds") metrics = self.reporter.metrics self.assertTrue(type(metrics) == type([])) self.assertTrue(len(metrics) > 0) self.assertEquals(len([t for t in metrics if t['metric'] == "tomcat.threads.busy" and "instance:tomcat_instance" in t['tags']]), 2, metrics) self.assertEquals(len([t for t in metrics if t['metric'] == "tomcat.bytes_sent" and "instance:tomcat_instance" in t['tags']]), 0, metrics) self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and "instance:tomcat_instance" in t['tags']]) > 4, metrics)
class JMXTestCase(unittest.TestCase): def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml') self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT}) self.t2 = threading.Thread(target=self.jmx_daemon.run) self.t2.start() def tearDown(self): self.server.stop() self.reporter.finished = True self.jmx_daemon.terminate() def testCustomJMXMetric(self): count = 0 while self.reporter.metrics is None: time.sleep(1) count += 1 if count > 20: raise Exception("No metrics were received in 20 seconds") metrics = self.reporter.metrics self.assertTrue(isinstance(metrics, ListType)) self.assertTrue(len(metrics) > 0) self.assertEquals(len([t for t in metrics if t['metric'] == "my.metric.buf" and "instance:jmx_instance1" in t['tags']]), 2, metrics) self.assertTrue(len([t for t in metrics if 'type:ThreadPool' in t['tags'] and "instance:jmx_instance1" in t['tags'] and "jmx.catalina" in t['metric']]) > 8, metrics) self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and "instance:jmx_instance1" in t['tags']]) == 13, metrics)
class JMXTestCase(unittest.TestCase): def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.join(os.path.dirname(__file__)) self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT}) self.t2 = threading.Thread(target=self.jmx_daemon.run) self.t2.start() def tearDown(self): self.server.stop() self.reporter.finished = True self.jmx_daemon.terminate() def testTomcatMetrics(self): count = 0 while self.reporter.metrics is None: time.sleep(1) count += 1 if count > 25: raise Exception("No metrics were received in 25 seconds") metrics = self.reporter.metrics self.assertTrue(isinstance(metrics, ListType)) self.assertTrue(len(metrics) > 8, metrics) self.assertEquals(len([t for t in metrics if 'instance:solr_instance' in t['tags'] and t['metric'] == "jvm.thread_count"]), 1, metrics) self.assertTrue(len([t for t in metrics if "jvm." in t['metric'] and 'instance:solr_instance' in t['tags']]) > 4, metrics) self.assertTrue(len([t for t in metrics if "solr." in t['metric'] and 'instance:solr_instance' in t['tags']]) > 4, metrics)
class JMXTestCase(unittest.TestCase): def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = Fixtures.directory() self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT}) self.t2 = threading.Thread(target=self.jmx_daemon.run) self.t2.start() def tearDown(self): self.server.stop() self.reporter.finished = True self.jmx_daemon.terminate() def testCustomJMXMetric(self): count = 0 while self.reporter.metrics is None: time.sleep(1) count += 1 if count > 25: raise Exception("No metrics were received in 25 seconds") metrics = self.reporter.metrics self.assertTrue(isinstance(metrics, ListType)) self.assertTrue(len(metrics) > 0) self.assertTrue(len([t for t in metrics if "cassandra.db." in t['metric'] and "instance:cassandra_instance" in t['tags']]) > 40, metrics)
class TestKafka(unittest.TestCase): """Basic Test for kafka integration.""" def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.join(os.path.dirname(__file__), 'resources/') self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT}) self.t2 = threading.Thread(target=self.jmx_daemon.run) self.t2.start() def tearDown(self): self.server.stop() self.reporter.finished = True self.jmx_daemon.terminate() def testCustomJMXMetric(self): count = 0 while self.reporter.metrics is None: time.sleep(1) count += 1 if count > 25: raise Exception("No metrics were received in 25 seconds") metrics = self.reporter.metrics # expected_tags = ['env:test', 'instance:kafka-172.17.0.1-9999', 'kafka:broker'] self.assertTrue(isinstance(metrics, ListType)) self.assertTrue(len(metrics) > 0) log.info(metrics) log.info(len(metrics)) self.assertTrue( len([t for t in metrics if "jvm." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) >= 13, metrics) self.assertTrue( len([t for t in metrics if "kafka.request." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) == 12, metrics) self.assertTrue( len([t for t in metrics if "kafka.replication." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) == 6, metrics) # CLIENT metrics. # kafka.producer.request_latency_avg self.assertTrue( len([t for t in metrics if "kafka.producer." in t['metric'] and "instance:kafka-172.17.0.1-7777" in t['tags']]) == 1, metrics) # kafka.consumer.fetch_rate, kafka.consumer.max_lag self.assertTrue( len([t for t in metrics if "kafka.consumer." in t['metric'] and "instance:kafka-172.17.0.1-7777" in t['tags']]) == 2, metrics)
class TestKafka(unittest.TestCase): """Basic Test for kafka integration.""" def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.join(os.path.dirname(__file__), 'ci/resources/') self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT}) self.t2 = threading.Thread(target=self.jmx_daemon.run) self.t2.start() def tearDown(self): self.server.stop() self.reporter.finished = True self.jmx_daemon.terminate() def testCustomJMXMetric(self): count = 0 while self.reporter.metrics is None: time.sleep(1) count += 1 if count > 25: raise Exception("No metrics were received in 25 seconds") metrics = self.reporter.metrics # expected_tags = ['env:test', 'instance:kafka-172.17.0.1-9999', 'kafka:broker'] self.assertTrue(isinstance(metrics, ListType)) self.assertTrue(len(metrics) > 0) log.info(metrics) log.info(len(metrics)) self.assertTrue( len([t for t in metrics if "jvm." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) >= 13, metrics) self.assertTrue( len([t for t in metrics if "kafka.request." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) == 12, metrics) self.assertTrue( len([t for t in metrics if "kafka.replication." in t['metric'] and "instance:kafka-172.17.0.1-9999" in t['tags']]) == 6, metrics) # CLIENT metrics. # kafka.producer.request_latency_avg self.assertTrue( len([t for t in metrics if "kafka.producer." in t['metric'] and "instance:kafka-172.17.0.1-7777" in t['tags']]) == 1, metrics) # kafka.consumer.fetch_rate, kafka.consumer.max_lag self.assertTrue( len([t for t in metrics if "kafka.consumer." in t['metric'] and "instance:kafka-172.17.0.1-7777" in t['tags']]) == 2, metrics)
class TestHbase_master(unittest.TestCase): """Basic Test for hbase_master integration.""" def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.join(os.path.dirname(__file__), 'ci/resources/') self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT}) self.t2 = threading.Thread(target=self.jmx_daemon.run) self.t2.start() def tearDown(self): self.server.stop() self.reporter.finished = True self.jmx_daemon.terminate() def testCustomJMXMetric(self): count = 0 while self.reporter.metrics is None: time.sleep(1) count += 1 if count > 60: raise Exception("No metrics were received in 60 seconds") metrics = self.reporter.metrics self.assertTrue(isinstance(metrics, ListType)) self.assertTrue(len(metrics) > 0) self.assertTrue( len([ t for t in metrics if "jvm." in t['metric'] and "instance:hbase_master-localhost-10101" in t['tags'] ]) >= 13, metrics) # 'Hadoop:service=HBase,name=Master,sub=Server - tag.isActiveMaster' needs to be converted to number. But 'values:'(valueConversion) doesn't work in yaml... self.assertTrue( len([ t for t in self.reporter.metrics if "hbase.master." in t['metric'] and "instance:hbase_master-localhost-10101" in t['tags'] ]) >= 20, metrics)
class TestHbase_regionserver(unittest.TestCase): """Basic Test for hbase_regionserver integration.""" def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.join(os.path.dirname(__file__), 'ci/resources/') self.jmx_daemon = JMXFetch(confd_path, {'dogstatsd_port': STATSD_PORT}) self.t2 = threading.Thread(target=self.jmx_daemon.run) self.t2.start() def tearDown(self): self.server.stop() self.reporter.finished = True self.jmx_daemon.terminate() def testCustomJMXMetric(self): count = 0 while self.reporter.metrics is None: time.sleep(1) count += 1 if count > 60: raise Exception("No metrics were received in 60 seconds") metrics = self.reporter.metrics self.assertTrue(isinstance(metrics, list)) self.assertTrue(len(metrics) > 0) self.assertTrue( len([ t for t in metrics if "jvm." in t['metric'] and "instance:hbase_regionserver-localhost-10102" in t['tags'] ]) >= 13, metrics) # waiting for receiving metrics which appears after a while. count = 0 while True: metrics = self.reporter.metrics mutations_metrics = [ t for t in metrics if "hbase.regionserver.server.mutations" in t['metric'] and "instance:hbase_regionserver-localhost-10102" in t['tags'] ] slow_appned_metrics = [ t for t in metrics if "hbase.regionserver.server.slow_append" in t['metric'] and "instance:hbase_regionserver-localhost-10102" in t['tags'] ] # hedged_metrics = [t for t in metrics if "hbase.regionserver.server.hedged_read" in t['metric'] and "instance:hbase_regionserver-localhost-10102" in t['tags']] # pause_time_metrics = [t for t in metrics if "hbase.regionserver.server.pause_time" in t['metric'] and "instance:hbase_regionserver-localhost-10102" in t['tags']] time.sleep(1) count += 1 if len(mutations_metrics) >= 2 and len(slow_appned_metrics) >= 1: break elif count <= 60: continue else: log.info(metrics) raise Exception("enough metrics were received in 60 seconds") # hbase.regionserver.server.hedged_read* and hbase.regionserver.server.pause* is ignored here # because these metrics won't emit until these process will actually happen. metrics = [ t for t in self.reporter.metrics if "hbase." in t['metric'] and "instance:hbase_regionserver-localhost-10102" in t['tags'] ] num_total = 159 num_hedged_read = 2 num_pause = 14 num = len(metrics) log.info(num) self.assertTrue(num >= (num_total - num_hedged_read - num_pause), metrics)