def test_update_partitions_without_zk(host):
"""
Check if can update partitions numbers without zk (only > 1.0.0)
"""
# Given
topic_name = get_topic_name()
ensure_kafka_topic(host, topic_defaut_configuration, topic_name)
time.sleep(0.3)
# When
test_topic_configuration = topic_defaut_configuration.copy()
test_topic_configuration.update({'partitions': 2})
ensure_idempotency(ensure_kafka_topic,
host,
test_topic_configuration,
topic_name,
minimal_api_version="1.0.0")
time.sleep(0.3)
# Then
for host, host_vars in kafka_hosts.items():
if (parse_version(
host_protocol_version[host_vars['inventory_hostname']]) <
parse_version("1.0.0")):
continue
kfk_addr = "%s:9092" % \
host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe']
check_configured_topic(host, test_topic_configuration, topic_name,
kfk_addr)
def test_delete_options(host):
"""
Check if can remove topic options
"""
# Given
init_topic_configuration = topic_defaut_configuration.copy()
init_topic_configuration.update(
{'options': {
'retention.ms': 66574936,
'flush.ms': 564939
}})
topic_name = get_topic_name()
ensure_kafka_topic(host, init_topic_configuration, topic_name)
time.sleep(0.3)
# When
test_topic_configuration = topic_defaut_configuration.copy()
test_topic_configuration.update({'options': {'flush.ms': 564939}})
ensure_idempotency(ensure_kafka_topic, host, test_topic_configuration,
topic_name)
time.sleep(0.3)
# Then
deleted_options = {
'retention.ms': 66574936,
}
for host, host_vars in kafka_hosts.items():
kfk_addr = "%s:9092" % \
host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe']
check_configured_topic(host,
test_topic_configuration,
topic_name,
kfk_addr,
deleted_options=deleted_options)
def test_update_replica_factor_force_reassign(host):
"""
Check if can update replication factor
"""
# Given
topic_name = get_topic_name()
ensure_kafka_topic(host, topic_defaut_configuration, topic_name)
time.sleep(0.3)
# When
test_topic_configuration = topic_defaut_configuration.copy()
test_topic_configuration.update({
'kafka_sleep_time': 10,
'kafka_max_retries': 10,
'replica_factor': 2,
'force_reassign': True,
'preserve_leader': True
})
ensure_kafka_topic_with_zk(host, test_topic_configuration, topic_name)
time.sleep(0.3)
test_topic_configuration.update({
'replica_factor': 2,
'force_reassign': True,
'preserve_leader': True
})
results = ensure_kafka_topic_with_zk(host, test_topic_configuration,
topic_name)
time.sleep(0.3)
# Then
for result in results:
assert result['changed'], str(result)
assert topic_name in result['changes']['topic_updated'], str(result)
def test_update_partitions():
"""
Check if can update partitions numbers
"""
# Given
topic_name = get_topic_name()
ensure_kafka_topic(
localhost,
topic_defaut_configuration,
topic_name
)
time.sleep(0.5)
# When
test_topic_configuration = topic_defaut_configuration.copy()
test_topic_configuration.update({
'partitions': 2
})
ensure_idempotency(
ensure_kafka_topic_with_zk,
localhost,
test_topic_configuration,
topic_name
)
time.sleep(0.5)
# Then
for host, host_vars in kafka_hosts.items():
kfk_addr = "%s:9092" % \
host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe']
check_configured_topic(host, test_topic_configuration,
topic_name, kfk_addr)
def test_delete_topic():
"""
Check if can delete topic
"""
# Given
topic_name = get_topic_name()
ensure_kafka_topic(
localhost,
topic_defaut_configuration,
topic_name
)
time.sleep(0.5)
# When
test_topic_configuration = topic_defaut_configuration.copy()
test_topic_configuration.update({
'state': 'absent'
})
ensure_idempotency(
ensure_kafka_topic,
localhost,
test_topic_configuration,
topic_name
)
time.sleep(0.5)
# Then
for host, host_vars in kafka_hosts.items():
kfk_addr = "%s:9092" % \
host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe']
check_configured_topic(host, test_topic_configuration,
topic_name, kfk_addr)
def test_check_mode():
"""
Check if can check mode do nothing
"""
# Given
topic_name = get_topic_name()
ensure_kafka_topic(
localhost,
topic_defaut_configuration,
topic_name
)
time.sleep(0.5)
# When
test_topic_configuration = topic_defaut_configuration.copy()
test_topic_configuration.update({
'state': 'absent'
})
ensure_kafka_topic_with_zk(
localhost,
test_topic_configuration,
topic_name,
check=True
)
time.sleep(0.5)
test_topic_configuration.update({
'state': 'present',
'partitions': topic_defaut_configuration['partitions'] + 1,
'replica_factor': topic_defaut_configuration['replica_factor'] + 1,
'options': {
'retention.ms': 1000
}
})
ensure_kafka_topic_with_zk(
localhost,
test_topic_configuration,
topic_name,
check=True
)
time.sleep(0.5)
new_topic_name = get_topic_name()
ensure_kafka_topic_with_zk(
localhost,
test_topic_configuration,
new_topic_name,
check=True
)
time.sleep(0.5)
# Then
expected_topic_configuration = topic_defaut_configuration.copy()
for host, host_vars in kafka_hosts.items():
kfk_addr = "%s:9092" % \
host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe']
check_configured_topic(host, expected_topic_configuration,
topic_name, kfk_addr)
test_topic_configuration.update({
'state': 'absent'
})
check_configured_topic(host, test_topic_configuration,
new_topic_name, kfk_addr)
def delete_consumer_offset(host, partitions=None):
"""
Check if can delete consumer group for topic
"""
# Given
consumer_group = get_consumer_group()
topic_name1 = get_topic_name()
ensure_kafka_topic(host,
topic_defaut_configuration,
topic_name1,
minimal_api_version="2.4.0")
time.sleep(0.3)
produce_and_consume_topic(topic_name1, 1, consumer_group, True, "2.4.0")
time.sleep(0.3)
# When
test_cg_configuration = cg_defaut_configuration.copy()
test_cg_configuration.update({
'consumer_group': consumer_group,
'action': 'delete',
'api_version': '2.4.0'
})
if partitions is None:
test_cg_configuration.update({'topics': [{'name': topic_name1}]})
else:
test_cg_configuration.update(
{'topics': [{
'name': topic_name1,
'partitions': partitions
}]})
test_cg_configuration.update(sasl_default_configuration)
ensure_idempotency(ensure_kafka_consumer_group,
host,
test_cg_configuration,
minimal_api_version="2.4.0")
time.sleep(0.3)
# Then
for host, host_vars in kafka_hosts.items():
if (parse_version(
host_protocol_version[host_vars['inventory_hostname']]) <
parse_version("2.4.0")):
continue
kfk_addr = "%s:9092" % \
host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe']
check_unconsumed_topic(consumer_group, topic_name1, kfk_addr)
def test_update_replica_factor(host):
"""
Check if can update replication factor
"""
# Given
topic_name = get_topic_name()
ensure_kafka_topic(host, topic_defaut_configuration, topic_name)
time.sleep(0.3)
# When
test_topic_configuration = topic_defaut_configuration.copy()
test_topic_configuration.update({'replica_factor': 2})
ensure_idempotency(ensure_kafka_topic_with_zk, host,
test_topic_configuration, topic_name)
time.sleep(0.3)
# Then
for host, host_vars in kafka_hosts.items():
kfk_addr = "%s:9092" % \
host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe']
check_configured_topic(host, test_topic_configuration, topic_name,
kfk_addr)
def test_update_partitions_and_replica_factor_default_value(host):
"""
Check if can update partitions numbers
make sure -1 values are considered, but warning + step is skipped
"""
# Given
topic_name = get_topic_name()
ensure_kafka_topic(host, topic_defaut_configuration, topic_name)
time.sleep(0.3)
# When
test_topic_configuration = topic_defaut_configuration.copy()
test_topic_configuration.update({'partitions': -1, 'replica_factor': -1})
ensure_kafka_topic(host, test_topic_configuration, topic_name)
time.sleep(0.3)
# Then
expected_topic_configuration = topic_defaut_configuration.copy()
expected_topic_configuration.update({'partitions': 1, 'replica_factor': 1})
for host, host_vars in kafka_hosts.items():
kfk_addr = "%s:9092" % \
host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe']
check_configured_topic(host, expected_topic_configuration, topic_name,
kfk_addr)
def test_rebalance_all_partitions_preserve_leader(host):
"""
Check if can reassign all topics-partition with
leader conservation.
"""
# Given
test_topic_name = get_topic_name()
test_topic_configuration = topic_defaut_configuration.copy()
test_topic_configuration.update({
'partitions': 50,
'replica_factor': 2,
'preserve_leader': True
})
ensure_kafka_topic(host, test_topic_configuration, test_topic_name)
time.sleep(0.3)
topics_config = call_kafka_info(
host, {
'resource': 'topic-config',
'include_internal': True,
'include_defaults': False
})
topics = call_kafka_info(host, {
'resource': 'topic',
'include_internal': True
})
all_topics = topics[0]['ansible_module_results']
all_topics_config = topics_config[0]['ansible_module_results']
test_topics = []
for topic_name, partitions in all_topics.items():
if topic_name == test_topic_name or topic_name.startswith('__'):
test_topics.append({
'name':
topic_name,
'partitions':
len(partitions),
'replica_factor':
len(partitions['0']['replicas']),
'options':
all_topics_config[topic_name],
'force_reassign':
True,
'preserve_leader':
True,
'state':
'present'
})
existing_leaders = []
for i, broker_topics in enumerate(topics):
existing_leaders.append({})
for topic_name, partitions in (
broker_topics['ansible_module_results'].items()):
if topic_name == test_topic_name or topic_name.startswith('__'):
for partition, config in partitions.items():
existing_leaders[i][(topic_name, partition)] = \
config['leader']
# When
test_topics_configuration = {
'topics': test_topics,
'kafka_sleep_time': 5,
'kafka_max_retries': 60
}
ensure_kafka_topics(host, test_topics_configuration)
time.sleep(10)
# Then
topics = call_kafka_info(host, {
'resource': 'topic',
'include_internal': True
})
for i, broker_topics in enumerate(topics):
resulting_replicas = {}
for topic_name, partitions in (
topics[i]['ansible_module_results'].items()):
if topic_name == test_topic_name or topic_name.startswith('__'):
for partition, config in partitions.items():
resulting_replicas[(topic_name, partition)] = \
config['replicas']
for topic_partition, leader in existing_leaders[i].items():
assert leader in resulting_replicas[topic_partition]
total_partitions = 0
partitions_per_broker = {}
for topic_name, partitions in (
topics[i]['ansible_module_results'].items()):
if topic_name == test_topic_name or topic_name.startswith('__'):
for partition, config in partitions.items():
for replica in config['replicas']:
if replica not in partitions_per_broker:
partitions_per_broker[replica] = 0
partitions_per_broker[replica] += 1
total_partitions += 1
mean_partitions = int(total_partitions * 1.0 /
len(partitions_per_broker))
for node_id, partitions in partitions_per_broker.items():
assert partitions >= mean_partitions, partitions_per_broker
assert partitions <= (mean_partitions + 1), partitions_per_broker
