def test_check_kafka(aggregator, kafka_instance):
"""
Testing Kafka_consumer check.
"""
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, {})
kafka_consumer_check.check(kafka_instance)
for name, consumer_group in iteritems(kafka_instance['consumer_groups']):
for topic, partitions in iteritems(consumer_group):
for partition in partitions:
tags = [
"topic:{}".format(topic), "partition:{}".format(partition)
] + ['optional:tag1']
for mname in BROKER_METRICS:
aggregator.assert_metric(mname, tags=tags, at_least=1)
for mname in CONSUMER_METRICS:
aggregator.assert_metric(
mname,
tags=tags +
["source:kafka", "consumer_group:{}".format(name)],
at_least=1)
# let's reassert for the __consumer_offsets - multiple partitions
aggregator.assert_metric('kafka.broker_offset', at_least=1)
aggregator.assert_all_metrics_covered()
def test_multiple_servers_zk(aggregator, zk_instance):
"""
Testing Kafka_consumer check.
"""
multiple_server_zk_instance = copy.deepcopy(zk_instance)
multiple_server_zk_instance['kafka_connect_str'] = [
multiple_server_zk_instance['kafka_connect_str'],
'{}:9092'.format(HOST),
]
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [{}])
kafka_consumer_check.check(multiple_server_zk_instance)
for name, consumer_group in multiple_server_zk_instance['consumer_groups'].items():
for topic, partitions in consumer_group.items():
for partition in partitions:
tags = ["topic:{}".format(topic), "partition:{}".format(partition)]
for mname in BROKER_METRICS:
aggregator.assert_metric(mname, tags=tags, at_least=1)
for mname in CONSUMER_METRICS:
aggregator.assert_metric(
mname, tags=tags + ["source:zk", "consumer_group:{}".format(name)], at_least=1
)
aggregator.assert_all_metrics_covered()
def test_check_nogroups_zk(aggregator, zk_instance):
"""
Testing Kafka_consumer check grabbing groups from ZK
"""
nogroup_instance = copy.deepcopy(zk_instance)
nogroup_instance.pop('consumer_groups')
nogroup_instance['monitor_unlisted_consumer_groups'] = True
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, {})
kafka_consumer_check.check(nogroup_instance)
for topic in TOPICS:
if topic != '__consumer_offsets':
for partition in PARTITIONS:
tags = [
"topic:{}".format(topic), "partition:{}".format(partition)
]
for mname in BROKER_METRICS:
aggregator.assert_metric(mname, tags=tags, at_least=1)
for mname in CONSUMER_METRICS:
aggregator.assert_metric(
mname,
tags=tags +
['source:zk', 'consumer_group:my_consumer'],
at_least=1)
else:
for mname in BROKER_METRICS + CONSUMER_METRICS:
aggregator.assert_metric(mname, at_least=1)
aggregator.assert_all_metrics_covered()
def test_uses_new_implementation_when_new_version_specified(kafka_instance):
instance = copy.deepcopy(kafka_instance)
instance['kafka_client_api_version'] = '0.10.2'
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [instance])
kafka_consumer_check._init_check_based_on_kafka_version()
assert isinstance(kafka_consumer_check.sub_check, NewKafkaConsumerCheck)
def test_can_send_event(aggregator, kafka_instance, dd_run_check):
"""
Testing Kafka_consumer check.
"""
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [kafka_instance])
kafka_consumer_check.send_event("test", "test", [], "test", "test")
aggregator.assert_event("test", exact_match=False, count=1)
def test_check_zk_basic_case_integration(aggregator, zk_instance):
"""
Testing Kafka_consumer check.
"""
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [zk_instance])
kafka_consumer_check.check(zk_instance)
_assert_check_zk_basic_case(aggregator, zk_instance)
def test_check_kafka(aggregator, kafka_instance):
"""
Testing Kafka_consumer check.
"""
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [kafka_instance])
kafka_consumer_check.check(kafka_instance)
assert_check_kafka(aggregator, kafka_instance['consumer_groups'])
def test_consumer_config_error(caplog):
instance = {
'kafka_connect_str': KAFKA_CONNECT_STR,
'kafka_consumer_offsets': True,
'tags': ['optional:tag1']
}
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [instance])
kafka_consumer_check.check(instance)
assert 'monitor_unlisted_consumer_groups is False' in caplog.text
def test_no_topics(aggregator, kafka_instance):
kafka_instance['consumer_groups'] = {'my_consumer': {}}
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [kafka_instance])
kafka_consumer_check.check(kafka_instance)
if is_legacy_check(kafka_consumer_check):
pytest.skip("This test does not apply to legacy check")
assert_check_kafka(aggregator, {'my_consumer': {'marvel': [0]}})
def test_check_kafka_metrics_limit(aggregator, kafka_instance):
"""
Testing Kafka_consumer check.
"""
kafka_consumer_check = KafkaCheck('kafka_consumer',
{'max_partition_contexts': 1},
[kafka_instance])
kafka_consumer_check.check(kafka_instance)
assert len(aggregator._metrics) == 1
def test_tls_config_legacy(extra_config, expected_http_kwargs, kafka_instance):
instance = kafka_instance
instance.update(extra_config)
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [instance])
kafka_consumer_check.get_tls_context()
actual_options = {
k: v
for k, v in kafka_consumer_check._tls_context_wrapper.config.items()
if k in expected_http_kwargs
}
assert expected_http_kwargs == actual_options
def test_consumer_config_error(caplog):
instance = {
'kafka_connect_str': KAFKA_CONNECT_STR,
'kafka_consumer_offsets': True,
'tags': ['optional:tag1']
}
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [instance])
if is_legacy_check(kafka_consumer_check):
pytest.skip("This test does not apply to legacy check")
kafka_consumer_check.check(instance)
assert 'monitor_unlisted_consumer_groups is False' in caplog.text
def test_version_metadata(datadog_agent, kafka_instance, dd_run_check):
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [kafka_instance])
kafka_consumer_check.check_id = 'test:123'
kafka_client = kafka_consumer_check.create_kafka_client()
version_data = [str(part) for part in kafka_client.check_version()]
kafka_client.close()
version_parts = {
'version.{}'.format(name): part
for name, part in zip(('major', 'minor', 'patch'), version_data)
}
version_parts['version.scheme'] = 'semver'
version_parts['version.raw'] = '.'.join(version_data)
dd_run_check(kafka_consumer_check)
datadog_agent.assert_metadata('test:123', version_parts)
def test_check_kafka(aggregator, kafka_instance):
"""
Testing Kafka_consumer check.
"""
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [kafka_instance])
kafka_consumer_check.check(kafka_instance)
for name, consumer_group in kafka_instance['consumer_groups'].items():
for topic, partitions in consumer_group.items():
for partition in partitions:
tags = ["topic:{}".format(topic), "partition:{}".format(partition)] + ['optional:tag1']
for mname in BROKER_METRICS:
aggregator.assert_metric(mname, tags=tags, at_least=1)
for mname in CONSUMER_METRICS:
aggregator.assert_metric(mname, tags=tags + ["consumer_group:{}".format(name)], at_least=1)
aggregator.assert_all_metrics_covered()
def test_tls_config_ok(kafka_instance_tls):
with mock.patch('datadog_checks.base.utils.tls.ssl') as ssl:
with mock.patch('kafka.KafkaClient') as kafka_client:
# mock Kafka Client
kafka_client.return_value = mock.MagicMock()
# mock TLS context
tls_context = mock.MagicMock()
ssl.SSLContext.return_value = tls_context
kafka_consumer_check = KafkaCheck('kafka_consumer', {},
[kafka_instance_tls])
kafka_consumer_check._create_kafka_client(clazz=kafka_client)
assert tls_context.check_hostname is True
assert tls_context.tls_cert is not None
assert tls_context.check_hostname is True
assert kafka_consumer_check.create_kafka_client is not None
def test_gssapi(kafka_instance, dd_run_check):
instance = copy.deepcopy(kafka_instance)
instance['kafka_client_api_version'] = '0.10.2'
instance['sasl_mechanism'] = 'GSSAPI'
instance['security_protocol'] = 'SASL_PLAINTEXT'
instance['sasl_kerberos_service_name'] = 'kafka'
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [instance])
# assert the check doesn't fail with:
# Exception: Could not find main GSSAPI shared library.
with pytest.raises(Exception, match='check_version'):
dd_run_check(kafka_consumer_check)
def test_multiple_servers_zk(kafka_cluster, kafka_producer, zk_consumer,
zk_instance, aggregator):
"""
Testing Kafka_consumer check.
"""
if not is_supported(['zookeeper']):
pytest.skip(
"zookeeper consumer offsets not supported in current environment")
if not kafka_producer.is_alive():
kafka_producer.start()
time.sleep(5)
if not zk_consumer.is_alive():
zk_consumer.start()
time.sleep(5)
multiple_server_zk_instance = copy.deepcopy(zk_instance)
multiple_server_zk_instance['kafka_connect_str'] = [
multiple_server_zk_instance['kafka_connect_str'],
'{}:9092'.format(HOST)
]
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, {})
kafka_consumer_check.check(multiple_server_zk_instance)
for name, consumer_group in multiple_server_zk_instance[
'consumer_groups'].iteritems():
for topic, partitions in consumer_group.iteritems():
for partition in partitions:
tags = [
"topic:{}".format(topic), "partition:{}".format(partition)
]
for mname in BROKER_METRICS:
aggregator.assert_metric(mname, tags=tags, at_least=1)
for mname in CONSUMER_METRICS:
aggregator.assert_metric(
mname,
tags=tags +
["source:zk", "consumer_group:{}".format(name)],
at_least=1)
def test_check_nogroups_zk(kafka_cluster, kafka_producer, zk_consumer,
zk_instance, aggregator):
"""
Testing Kafka_consumer check grabbing groups from ZK
"""
if not is_supported(['zookeeper']):
pytest.skip(
"zookeeper consumer offsets not supported in current environment")
if not kafka_producer.is_alive():
kafka_producer.start()
time.sleep(5)
if not zk_consumer.is_alive():
zk_consumer.start()
time.sleep(5)
nogroup_instance = copy.deepcopy(zk_instance)
nogroup_instance.pop('consumer_groups')
nogroup_instance['monitor_unlisted_consumer_groups'] = True
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, {})
kafka_consumer_check.check(nogroup_instance)
for topic in TOPICS:
if topic is not '__consumer_offsets':
for partition in PARTITIONS:
tags = [
"topic:{}".format(topic), "partition:{}".format(partition)
]
for mname in BROKER_METRICS:
aggregator.assert_metric(mname, tags=tags, at_least=1)
for mname in CONSUMER_METRICS:
aggregator.assert_metric(
mname,
tags=tags +
["source:zk", "consumer_group:my_consumer"],
at_least=1)
else:
for mname in BROKER_METRICS + CONSUMER_METRICS:
aggregator.assert_metric(mname, at_least=1)
def test_get_interpolated_timestamp(kafka_instance):
instance = copy.deepcopy(kafka_instance)
instance['kafka_client_api_version'] = '0.10.2'
instance['sasl_kerberos_service_name'] = 'kafka'
check = KafkaCheck('kafka_consumer', {}, [instance])
check._init_check_based_on_kafka_version()
# at offset 0, time is 100s, at offset 10, time is 200sec.
# by interpolation, at offset 5, time should be 150sec.
assert check.sub_check._get_interpolated_timestamp({
0: 100,
10: 200
}, 5) == 150
assert check.sub_check._get_interpolated_timestamp({
10: 100,
20: 200
}, 5) == 50
assert check.sub_check._get_interpolated_timestamp({
0: 100,
10: 200
}, 15) == 250
assert check.sub_check._get_interpolated_timestamp({10: 200}, 15) is None
def test_check_zk(aggregator, zk_instance):
"""
Testing Kafka_consumer check.
"""
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, {})
kafka_consumer_check.check(zk_instance)
for name, consumer_group in iteritems(zk_instance['consumer_groups']):
for topic, partitions in iteritems(consumer_group):
for partition in partitions:
tags = ["topic:{}".format(topic), "partition:{}".format(partition)]
for mname in BROKER_METRICS:
aggregator.assert_metric(mname, tags=tags, at_least=1)
for mname in CONSUMER_METRICS:
aggregator.assert_metric(
mname, tags=tags + ["source:zk", "consumer_group:{}".format(name)], at_least=1
)
# let's reassert for the __consumer_offsets - multiple partitions
aggregator.assert_metric('kafka.broker_offset', at_least=1)
aggregator.assert_all_metrics_covered()
all_partitions = {
'kafka_connect_str': KAFKA_CONNECT_STR,
'zk_connect_str': ZK_CONNECT_STR,
'consumer_groups': {'my_consumer': {'marvel': []}},
}
kafka_consumer_check.check(all_partitions)
aggregator.assert_metric(
'kafka.consumer_offset', tags=['topic:marvel', 'partition:0',
'consumer_group:my_consumer', 'source:zk'], at_least=1)
aggregator.assert_metric(
'kafka.consumer_offset', tags=['topic:marvel', 'partition:1',
'consumer_group:my_consumer', 'source:zk'], at_least=1)
def test_check_no_partitions_zk(aggregator, zk_instance):
"""
Testing Kafka_consumer check grabbing partitions from ZK
"""
no_partitions_instance = copy.deepcopy(zk_instance)
topic = 'marvel'
no_partitions_instance['consumer_groups'] = {'my_consumer': {topic: []}}
kafka_consumer_check = KafkaCheck('kafka_consumer', {},
[no_partitions_instance])
kafka_consumer_check.check(no_partitions_instance)
for partition in PARTITIONS:
tags = ["topic:{}".format(topic), "partition:{}".format(partition)]
for mname in BROKER_METRICS:
aggregator.assert_metric(mname, tags=tags, at_least=1)
for mname in CONSUMER_METRICS:
aggregator.assert_metric(
mname,
tags=tags + ['source:zk', 'consumer_group:my_consumer'],
at_least=1)
aggregator.assert_all_metrics_covered()
def test_check_zk(kafka_cluster, kafka_producer, zk_consumer, zk_instance,
aggregator):
"""
Testing Kafka_consumer check.
"""
if not is_supported(['zookeeper']):
pytest.skip(
"zookeeper consumer offsets not supported in current environment")
if not kafka_producer.is_alive():
kafka_producer.start()
time.sleep(5)
if not zk_consumer.is_alive():
zk_consumer.start()
time.sleep(5)
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, {})
kafka_consumer_check.check(zk_instance)
for name, consumer_group in zk_instance['consumer_groups'].iteritems():
for topic, partitions in consumer_group.iteritems():
for partition in partitions:
tags = [
"topic:{}".format(topic), "partition:{}".format(partition)
]
for mname in BROKER_METRICS:
aggregator.assert_metric(mname, tags=tags, at_least=1)
for mname in CONSUMER_METRICS:
aggregator.assert_metric(
mname,
tags=tags +
["source:zk", "consumer_group:{}".format(name)],
at_least=1)
# let's reassert for the __consumer_offsets - multiple partitions
aggregator.assert_metric('kafka.broker_offset', at_least=1)
def test_should_zk():
check = KafkaCheck('kafka_consumer', {}, {})
# Kafka Consumer Offsets set to True and we have a zk_connect_str that hasn't been run yet
assert check._should_zk([ZK_CONNECT_STR, ZK_CONNECT_STR], 10, True) is True
# Kafka Consumer Offsets is set to False, should immediately ZK
assert check._should_zk(ZK_CONNECT_STR, 10, False) is True
# Last time we checked ZK_CONNECT_STR was less than interval ago, shouldn't ZK
zk_connect_hash = hash_mutable(ZK_CONNECT_STR)
check._zk_last_ts[zk_connect_hash] = time.time()
assert check._should_zk(ZK_CONNECT_STR, 100, True) is False
def test_no_partitions(aggregator, kafka_instance):
kafka_instance['consumer_groups'] = {'my_consumer': {'marvel': []}}
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [kafka_instance])
kafka_consumer_check.check(kafka_instance)
assert_check_kafka(aggregator, {'my_consumer': {'marvel': [0]}})
def test_no_topics(aggregator, kafka_instance, dd_run_check):
kafka_instance['consumer_groups'] = {'my_consumer': {}}
kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [kafka_instance])
dd_run_check(kafka_consumer_check)
assert_check_kafka(aggregator, {'my_consumer': {'marvel': [0]}})
