def setUp(self):
config = get_collector_config('KafkaCollector', {
'interval': 10
})
self.collector = KafkaCollector(config, None)
class TestKafkaCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('KafkaCollector', {
'interval': 10
})
self.collector = KafkaCollector(config, None)
def _get_xml_fixture(self, name):
fixture = self.getFixture(name)
return ElementTree.fromstring(fixture.getvalue())
def test_import(self):
self.assertTrue(KafkaCollector)
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
def test_get(self, urlopen_mock):
urlopen_mock.return_value = self.getFixture('empty.xml')
result = self.collector._get('/path')
result_string = ElementTree.tostring(result)
self.assertEqual(result_string, '<Server />')
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
def test_get_httperror(self, urlopen_mock):
urlopen_mock.side_effect = urllib2.URLError('BOOM')
result = self.collector._get('/path')
self.assertFalse(result)
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
def test_get_bad_xml(self, urlopen_mock):
urlopen_mock.return_value = self.getFixture('bad.xml')
result = self.collector._get('/path')
self.assertFalse(result)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_get_mbeans(self, get_mock):
get_mock.return_value = self._get_xml_fixture('serverbydomain.xml')
expected_names = set([
'kafka:type=kafka.BrokerAllTopicStat',
'kafka:type=kafka.BrokerTopicStat.mytopic',
'kafka:type=kafka.LogFlushStats',
'kafka:type=kafka.SocketServerStats',
'kafka:type=kafka.logs.mytopic-0',
'kafka:type=kafka.logs.mytopic-1',
'kafka:type=kafka.Log4jController',
])
found_beans = self.collector.get_mbeans('*')
self.assertEqual(found_beans, expected_names)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_get_mbeans_get_fail(self, get_mock):
get_mock.return_value = None
found_beans = self.collector.get_mbeans('*')
self.assertEqual(found_beans, None)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_query_mbean(self, get_mock):
get_mock.return_value = self._get_xml_fixture('mbean.xml')
expected_metrics = {
'kafka.logs.mytopic-1.CurrentOffset': long('213500615'),
'kafka.logs.mytopic-1.NumAppendedMessages': long('224634137'),
'kafka.logs.mytopic-1.NumberOfSegments': int('94'),
'kafka.logs.mytopic-1.Size': long('50143615339'),
}
metrics = self.collector.query_mbean('kafka:type=kafka.logs.mytopic-1')
self.assertEqual(metrics, expected_metrics)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_query_mbean_with_prefix(self, get_mock):
get_mock.return_value = self._get_xml_fixture('mbean.xml')
expected_metrics = {
'some.prefix.CurrentOffset': long('213500615'),
'some.prefix.NumAppendedMessages': long('224634137'),
'some.prefix.NumberOfSegments': int('94'),
'some.prefix.Size': long('50143615339'),
}
metrics = self.collector.query_mbean('kafka:type=kafka.logs.mytopic-0',
'some.prefix')
self.assertEqual(metrics, expected_metrics)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_query_mbean_fail(self, get_mock):
get_mock.return_value = None
metrics = self.collector.query_mbean('kafka:type=kafka.logs.mytopic-0')
self.assertEqual(metrics, None)
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
@patch.object(Collector, 'publish')
def test(self, publish_mock, urlopen_mock):
urlopen_mock.side_effect = [
self.getFixture('serverbydomain_logs_only.xml'),
self.getFixture('serverbydomain_gc.xml'),
self.getFixture('serverbydomain_threading.xml'),
self.getFixture('gc_scavenge.xml'),
self.getFixture('gc_marksweep.xml'),
self.getFixture('mbean.xml'),
self.getFixture('threading.xml'),
]
self.collector.collect()
expected_metrics = {
'kafka.logs.mytopic-1.CurrentOffset': 213500615,
'kafka.logs.mytopic-1.NumAppendedMessages': 224634137,
'kafka.logs.mytopic-1.NumberOfSegments': 94,
'kafka.logs.mytopic-1.Size': 50143615339,
'Threading.CurrentThreadCpuTime': 0,
'Threading.CurrentThreadUserTime': 0,
'Threading.DaemonThreadCount': 58,
'Threading.PeakThreadCount': 90,
'Threading.ThreadCount': 89,
'Threading.TotalStartedThreadCount': 228,
'GarbageCollector.PSScavenge.CollectionCount': 37577,
'GarbageCollector.PSScavenge.CollectionTime': 112293,
'GarbageCollector.PSMarkSweep.CollectionCount': 2,
'GarbageCollector.PSMarkSweep.CollectionTime': 160,
}
self.assertPublishedMany(publish_mock, expected_metrics)
class TestKafkaCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('KafkaCollector', {'interval': 10})
self.collector = KafkaCollector(config, None)
def _get_xml_fixture(self, name):
fixture = self.getFixture(name)
return ElementTree.fromstring(fixture.getvalue())
def test_import(self):
self.assertTrue(KafkaCollector)
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
def test_get(self, urlopen_mock):
urlopen_mock.return_value = self.getFixture('empty.xml')
result = self.collector._get('/path')
result_string = ElementTree.tostring(result)
self.assertEqual(result_string, '<Server />')
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
def test_get_httperror(self, urlopen_mock):
urlopen_mock.side_effect = urllib2.URLError('BOOM')
result = self.collector._get('/path')
self.assertFalse(result)
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
def test_get_bad_xml(self, urlopen_mock):
urlopen_mock.return_value = self.getFixture('bad.xml')
result = self.collector._get('/path')
self.assertFalse(result)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_get_mbeans(self, get_mock):
get_mock.return_value = self._get_xml_fixture('serverbydomain.xml')
expected_names = {
'kafka:type=kafka.BrokerAllTopicStat',
'kafka:type=kafka.BrokerTopicStat.mytopic',
'kafka:type=kafka.LogFlushStats',
'kafka:type=kafka.SocketServerStats',
'kafka:type=kafka.logs.mytopic-0',
'kafka:type=kafka.logs.mytopic-1',
'kafka:type=kafka.Log4jController'
}
found_beans = self.collector.get_mbeans('*')
self.assertEqual(found_beans, expected_names)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_get_mbeans_get_fail(self, get_mock):
get_mock.return_value = None
found_beans = self.collector.get_mbeans('*')
self.assertEqual(found_beans, None)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_query_mbean(self, get_mock):
get_mock.return_value = self._get_xml_fixture('mbean.xml')
expected_metrics = {
'kafka.logs.mytopic-1.CurrentOffset': long('213500615'),
'kafka.logs.mytopic-1.NumAppendedMessages': long('224634137'),
'kafka.logs.mytopic-1.NumberOfSegments': int('94'),
'kafka.logs.mytopic-1.Size': long('50143615339'),
}
metrics = self.collector.query_mbean('kafka:type=kafka.logs.mytopic-1')
self.assertEqual(metrics, expected_metrics)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_query_mbean_with_prefix(self, get_mock):
get_mock.return_value = self._get_xml_fixture('mbean.xml')
expected_metrics = {
'some.prefix.CurrentOffset': long('213500615'),
'some.prefix.NumAppendedMessages': long('224634137'),
'some.prefix.NumberOfSegments': int('94'),
'some.prefix.Size': long('50143615339'),
}
metrics = self.collector.query_mbean('kafka:type=kafka.logs.mytopic-0',
'some.prefix')
self.assertEqual(metrics, expected_metrics)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_activeController_value(self, get_mock):
get_mock.return_value = self._get_xml_fixture(
'activecontrollercount.xml')
expected_metrics = {
'KafkaController.ActiveControllerCount.Value': 1.0,
}
metrics = self.collector.query_mbean(
'kafka.controller:type=KafkaController,name=ActiveControllerCount')
self.assertEqual(metrics, expected_metrics)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_query_mbean_fail(self, get_mock):
get_mock.return_value = None
metrics = self.collector.query_mbean('kafka:type=kafka.logs.mytopic-0')
self.assertEqual(metrics, None)
def getKafkaFixture(self, url):
url_object = urlparse(url)
query_string = parse_qs(url_object.query)
querynames = query_string.get('querynames', [])
objectnames = query_string.get('objectname', [])
if url_object.path == '/serverbydomain':
if 'java.lang:type=GarbageCollector,name=*' in querynames:
return self.getFixture('serverbydomain_gc.xml')
elif '*kafka.controller:*' in querynames:
return self.getFixture('kafkacontroller.xml')
elif 'java.lang:type=Threading' in querynames:
return self.getFixture('serverbydomain_threading.xml')
else:
return self.getFixture('serverbydomain_logs_only.xml')
elif url_object.path == '/mbean':
if ('java.lang:type=GarbageCollector,name=PS MarkSweep'
in objectnames):
return self.getFixture('gc_marksweep.xml')
elif ('kafka.controller:type=KafkaController,' +
'name=ActiveControllerCount' in objectnames):
return self.getFixture('activecontrollercount.xml')
elif ('java.lang:type=GarbageCollector,name=PS Scavenge'
in objectnames):
return self.getFixture('gc_scavenge.xml')
elif 'java.lang:type=Threading' in objectnames:
return self.getFixture('threading.xml')
else:
return self.getFixture('mbean.xml')
else:
return ''
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
@patch.object(Collector, 'publish')
def test(self, publish_mock, urlopen_mock):
urlopen_mock.side_effect = self.getKafkaFixture
self.collector.collect()
expected_metrics = {
'kafka.logs.mytopic-1.CurrentOffset': 213500615,
'kafka.logs.mytopic-1.NumAppendedMessages': 224634137,
'kafka.logs.mytopic-1.NumberOfSegments': 94,
'kafka.logs.mytopic-1.Size': 50143615339,
'Threading.CurrentThreadCpuTime': 0,
'Threading.CurrentThreadUserTime': 0,
'Threading.DaemonThreadCount': 58,
'Threading.PeakThreadCount': 90,
'Threading.ThreadCount': 89,
'Threading.TotalStartedThreadCount': 228,
'GarbageCollector.PSScavenge.CollectionCount': 37577,
'GarbageCollector.PSScavenge.CollectionTime': 112293,
'GarbageCollector.PSMarkSweep.CollectionCount': 2,
'GarbageCollector.PSMarkSweep.CollectionTime': 160,
}
self.assertPublishedMany(publish_mock, expected_metrics)
def setUp(self):
config = get_collector_config('KafkaCollector', {'interval': 10})
self.collector = KafkaCollector(config, None)
class TestKafkaCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('KafkaCollector', {'interval': 10})
self.collector = KafkaCollector(config, None)
def _get_xml_fixture(self, name):
fixture = self.getFixture(name)
return ElementTree.fromstring(fixture.getvalue())
def test_import(self):
self.assertTrue(KafkaCollector)
def _verify_calls(self, actual, expected):
assert len(actual) == len(expected)
for call in actual:
curr = (call[0][0].split('.')[-1], call[0][1])
assert curr in expected
expected.remove(curr)
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
def test_get(self, urlopen_mock):
urlopen_mock.return_value = self.getFixture('empty.xml')
result = self.collector._get('/path')
result_string = ElementTree.tostring(result)
self.assertEqual(result_string, '<Server />')
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
def test_get_httperror(self, urlopen_mock):
urlopen_mock.side_effect = urllib2.URLError('BOOM')
result = self.collector._get('/path')
self.assertFalse(result)
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
def test_get_bad_xml(self, urlopen_mock):
urlopen_mock.return_value = self.getFixture('bad.xml')
result = self.collector._get('/path')
self.assertFalse(result)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_get_mbeans(self, get_mock):
get_mock.return_value = self._get_xml_fixture('serverbydomain.xml')
expected_names = set([
'kafka:type=kafka.BrokerAllTopicStat',
'kafka:type=kafka.BrokerTopicStat.mytopic',
'kafka:type=kafka.LogFlushStats',
'kafka:type=kafka.SocketServerStats',
'kafka:type=kafka.logs.mytopic-0',
'kafka:type=kafka.logs.mytopic-1',
'kafka:type=kafka.Log4jController',
])
found_beans = self.collector.get_mbeans('*')
self.assertEqual(found_beans, expected_names)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_get_mbeans_get_fail(self, get_mock):
get_mock.return_value = None
found_beans = self.collector.get_mbeans('*')
self.assertEqual(found_beans, None)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_query_mbean(self, get_mock):
get_mock.return_value = self._get_xml_fixture('mbean.xml')
expected_metrics = {
'kafka.logs.mytopic-1.CurrentOffset': long('213500615'),
'kafka.logs.mytopic-1.NumAppendedMessages': long('224634137'),
'kafka.logs.mytopic-1.NumberOfSegments': int('94'),
'kafka.logs.mytopic-1.Size': long('50143615339'),
}
metrics = self.collector.query_mbean('kafka:type=kafka.logs.mytopic-1')
self.assertEqual(metrics, expected_metrics)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_query_mbean_with_prefix(self, get_mock):
get_mock.return_value = self._get_xml_fixture('mbean.xml')
expected_metrics = {
'some.prefix.CurrentOffset': long('213500615'),
'some.prefix.NumAppendedMessages': long('224634137'),
'some.prefix.NumberOfSegments': int('94'),
'some.prefix.Size': long('50143615339'),
}
metrics = self.collector.query_mbean('kafka:type=kafka.logs.mytopic-0',
'some.prefix')
self.assertEqual(metrics, expected_metrics)
@run_only_if_ElementTree_is_available
@patch.object(KafkaCollector, '_get')
def test_query_mbean_fail(self, get_mock):
get_mock.return_value = None
metrics = self.collector.query_mbean('kafka:type=kafka.logs.mytopic-0')
self.assertEqual(metrics, None)
@run_only_if_ElementTree_is_available
@patch('urllib2.urlopen')
@patch.object(Collector, 'publish')
def test(self, publish_mock, urlopen_mock):
urlopen_mock.side_effect = [
self.getFixture('serverbydomain_logs_only.xml'),
self.getFixture('serverbydomain_gc.xml'),
self.getFixture('serverbydomain_threading.xml'),
self.getFixture('gc_scavenge.xml'),
self.getFixture('gc_marksweep.xml'),
self.getFixture('mbean.xml'),
self.getFixture('threading.xml'),
]
self.collector.collect()
# The metrics emitted are slightly inconsistent, but this ensures
# the latter half are correct.
expected_metrics = [
('CurrentOffset', 213500615L),
('NumAppendedMessages', 224634137L),
('NumberOfSegments', 94),
('Size', 50143615339L),
('CurrentThreadCpuTime', 0L),
('CurrentThreadUserTime', 0L),
('DaemonThreadCount', 58),
('PeakThreadCount', 90),
('ThreadCount', 89),
('TotalStartedThreadCount', 228L),
('CollectionCount', 2L),
('CollectionTime', 112293L),
('CollectionTime', 160L),
('CollectionCount', 37577L),
]
calls = publish_mock.call_args_list
self._verify_calls(calls, expected_metrics)
