def test_check_mode(): """ Check if can check mode do nothing """ # Given topic_name = get_topic_name() ensure_kafka_topic( localhost, topic_defaut_configuration, topic_name ) time.sleep(0.5) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'state': 'absent' }) ensure_kafka_topic_with_zk( localhost, test_topic_configuration, topic_name, check=True ) time.sleep(0.5) test_topic_configuration.update({ 'state': 'present', 'partitions': topic_defaut_configuration['partitions'] + 1, 'replica_factor': topic_defaut_configuration['replica_factor'] + 1, 'options': { 'retention.ms': 1000 } }) ensure_kafka_topic_with_zk( localhost, test_topic_configuration, topic_name, check=True ) time.sleep(0.5) new_topic_name = get_topic_name() ensure_kafka_topic_with_zk( localhost, test_topic_configuration, new_topic_name, check=True ) time.sleep(0.5) # Then expected_topic_configuration = topic_defaut_configuration.copy() for host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(host, expected_topic_configuration, topic_name, kfk_addr) test_topic_configuration.update({ 'state': 'absent' }) check_configured_topic(host, test_topic_configuration, new_topic_name, kfk_addr)
def test_kafka_info_topic(host): """ Check if can get info on topic """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) topic_test_name = get_topic_name() topic_test_configuration = topic_defaut_configuration.copy() topic_test_configuration.update({ 'options': { 'min.insync.replicas': 2 } }) ensure_topic( host, topic_test_configuration, topic_test_name ) time.sleep(0.3) produce_and_consume_topic(topic_name, 10, get_consumer_group()) time.sleep(0.3) # When results = call_kafka_info( host, { 'resource': 'topic' } ) # Then for r in results: assert topic_name in r['ansible_module_results'] assert '__consumer_offsets' not in r['ansible_module_results'] for name, topic_info in r['ansible_module_results'].items(): for partition, partition_info in topic_info.items(): assert 'earliest_offset' in partition_info assert 'latest_offset' in partition_info assert partition_info['earliest_offset'] >= 0 assert (partition_info['latest_offset'] == 10 if name == topic_name else partition_info['latest_offset'] >= 0) if name == topic_name: assert partition_info['at_min_isr'] assert not partition_info['under_replicated'] assert not partition_info['under_min_isr'] assert not partition_info['unavailable_partition'] elif name == topic_test_name: assert not partition_info['at_min_isr'] assert not partition_info['under_replicated'] assert partition_info['under_min_isr'] assert not partition_info['unavailable_partition']
def test_update_replica_factor(): """ Check if can update replication factor """ # Given topic_name = get_topic_name() ensure_kafka_topic( localhost, topic_defaut_configuration, topic_name ) time.sleep(0.5) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'replica_factor': 2 }) ensure_idempotency( ensure_kafka_topic_with_zk, localhost, test_topic_configuration, topic_name ) time.sleep(0.5) # Then for host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(host, test_topic_configuration, topic_name, kfk_addr)
def test_delete_topic(): """ Check if can delete topic """ # Given topic_name = get_topic_name() ensure_kafka_topic( localhost, topic_defaut_configuration, topic_name ) time.sleep(0.5) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'state': 'absent' }) ensure_idempotency( ensure_kafka_topic, localhost, test_topic_configuration, topic_name ) time.sleep(0.5) # Then for host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(host, test_topic_configuration, topic_name, kfk_addr)
def test_consumer_lag(host): """ Check if can check global consumer lag """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) consumer_group = get_consumer_group() total_msg = 42 # When produce_and_consume_topic( topic_name, total_msg, consumer_group) time.sleep(0.3) # Then lags = call_kafka_stat_lag(host, { 'consummer_group': consumer_group }) for lag in lags: msg = json.loads(lag['msg']) global_lag_count = msg['global_lag_count'] assert global_lag_count == (total_msg - 1)
def test_kafka_info_topics_config_not_include_defaults(host): """ Check if can get config on topic. """ # Given topic_name = get_topic_name() test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'options': { 'retention.ms': 66574936 } }) ensure_topic( host, test_topic_configuration, topic_name ) time.sleep(0.3) # When results = call_kafka_info( host, { 'resource': 'topic-config', 'include_defaults': False }, ) # Then for r in results: assert topic_name in r['ansible_module_results'] for name, topic_config in r['ansible_module_results'].items(): if name == topic_name: assert int(topic_config['retention.ms']) == 66574936 assert 'min.insync.replicas' not in topic_config
def test_update_partitions_without_zk(host): """ Check if can update partitions numbers without zk (only > 1.0.0) """ # Given topic_name = get_topic_name() ensure_kafka_topic(host, topic_defaut_configuration, topic_name) time.sleep(0.3) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({'partitions': 2}) ensure_idempotency(ensure_kafka_topic, host, test_topic_configuration, topic_name, minimal_api_version="1.0.0") time.sleep(0.3) # Then for host, host_vars in kafka_hosts.items(): if (parse_version( host_protocol_version[host_vars['inventory_hostname']]) < parse_version("1.0.0")): continue kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(host, test_topic_configuration, topic_name, kfk_addr)
def test_update_partitions(host): """ Check if can update partitions numbers """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'partitions': 2 }) ensure_idempotency( ensure_topic, host, test_topic_configuration, topic_name ) time.sleep(0.3) # Then for kafka_host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(kafka_host, test_topic_configuration, topic_name, kfk_addr)
def test_duplicated_topics(host): """ Check if can remove topics options """ # Given duplicated_topic_name = get_topic_name() def get_topic_config(): topic_configuration = topic_defaut_configuration.copy() topic_configuration.update({ 'name': duplicated_topic_name, 'options': { 'retention.ms': 66574936, 'flush.ms': 564939 } }) return topic_configuration topic_configuration = {'topics': [get_topic_config(), get_topic_config()]} # When results = ensure_kafka_topics(host, topic_configuration) time.sleep(0.3) # Then for result in results: assert not result['changed'] assert 'duplicated topics' in result['msg']
def test_delete_options(host): """ Check if can remove topic options """ # Given init_topic_configuration = topic_defaut_configuration.copy() init_topic_configuration.update( {'options': { 'retention.ms': 66574936, 'flush.ms': 564939 }}) topic_name = get_topic_name() ensure_kafka_topic(host, init_topic_configuration, topic_name) time.sleep(0.3) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({'options': {'flush.ms': 564939}}) ensure_idempotency(ensure_kafka_topic, host, test_topic_configuration, topic_name) time.sleep(0.3) # Then deleted_options = { 'retention.ms': 66574936, } for host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(host, test_topic_configuration, topic_name, kfk_addr, deleted_options=deleted_options)
def test_update_replica_factor_force_reassign(host): """ Check if can update replication factor """ # Given topic_name = get_topic_name() ensure_kafka_topic(host, topic_defaut_configuration, topic_name) time.sleep(0.3) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'kafka_sleep_time': 10, 'kafka_max_retries': 10, 'replica_factor': 2, 'force_reassign': True, 'preserve_leader': True }) ensure_kafka_topic_with_zk(host, test_topic_configuration, topic_name) time.sleep(0.3) test_topic_configuration.update({ 'replica_factor': 2, 'force_reassign': True, 'preserve_leader': True }) results = ensure_kafka_topic_with_zk(host, test_topic_configuration, topic_name) time.sleep(0.3) # Then for result in results: assert result['changed'], str(result) assert topic_name in result['changes']['topic_updated'], str(result)
def test_kafka_info_consumer_group(host): """ Check if can get info on consumer groups """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) consumer_group = get_consumer_group() total_msg = 42 # When produce_and_consume_topic( topic_name, total_msg, consumer_group) time.sleep(0.3) # Then results = call_kafka_info( host, { 'resource': 'consumer_group' } ) for r in results: assert consumer_group in r['ansible_module_results']
def get_topic_config(): topic_configuration = topic_defaut_configuration.copy() topic_configuration.update({ 'name': get_topic_name(), 'options': { 'retention.ms': 66574936, 'flush.ms': 564939 } }) return topic_configuration
def test_kafka_info_topic_include_internal(host): """ Check if can get info on topic """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) topic_test_name = get_topic_name() topic_test_configuration = topic_defaut_configuration.copy() topic_test_configuration.update({ 'options': { 'min.insync.replicas': 2 } }) ensure_topic( host, topic_test_configuration, topic_test_name ) time.sleep(0.3) produce_and_consume_topic(topic_name, 10, get_consumer_group()) time.sleep(0.3) # When results = call_kafka_info( host, { 'resource': 'topic', 'include_internal': True } ) # Then for r in results: assert topic_name in r['ansible_module_results'] assert '__consumer_offsets' in r['ansible_module_results']
def test_kafka_info_topic(host): """ Check if can get info on topic """ # Given topic_name = get_topic_name() ensure_topic(host, topic_defaut_configuration, topic_name) time.sleep(0.3) # When results = call_kafka_info(host, {'resource': 'topic'}) # Then for r in results: assert topic_name in r['ansible_module_results']
def test_kafka_info_brokers(): """ Check if can get info on brokers """ # Given topic_name = get_topic_name() ensure_topic(localhost, topic_defaut_configuration, topic_name) time.sleep(0.5) # When results = call_kafka_info(localhost, {'resource': 'broker'}) # Then for r in results: assert len(r['ansible_module_results']) == 2
def delete_consumer_offset(host, partitions=None): """ Check if can delete consumer group for topic """ # Given consumer_group = get_consumer_group() topic_name1 = get_topic_name() ensure_kafka_topic(host, topic_defaut_configuration, topic_name1, minimal_api_version="2.4.0") time.sleep(0.3) produce_and_consume_topic(topic_name1, 1, consumer_group, True, "2.4.0") time.sleep(0.3) # When test_cg_configuration = cg_defaut_configuration.copy() test_cg_configuration.update({ 'consumer_group': consumer_group, 'action': 'delete', 'api_version': '2.4.0' }) if partitions is None: test_cg_configuration.update({'topics': [{'name': topic_name1}]}) else: test_cg_configuration.update( {'topics': [{ 'name': topic_name1, 'partitions': partitions }]}) test_cg_configuration.update(sasl_default_configuration) ensure_idempotency(ensure_kafka_consumer_group, host, test_cg_configuration, minimal_api_version="2.4.0") time.sleep(0.3) # Then for host, host_vars in kafka_hosts.items(): if (parse_version( host_protocol_version[host_vars['inventory_hostname']]) < parse_version("2.4.0")): continue kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_unconsumed_topic(consumer_group, topic_name1, kfk_addr)
def test_consumer_lag(): """ Check if can check global consumer lag """ # Given topic_name = get_topic_name() ensure_topic(localhost, topic_defaut_configuration, topic_name) time.sleep(0.5) consumer_group = get_consumer_group() total_msg = 42 # When produce_and_consume_topic(topic_name, total_msg, consumer_group) time.sleep(0.5) # Then for host, host_vars in kafka_hosts.items(): lag = call_kafka_stat_lag(localhost, {'consummer_group': consumer_group}) msg = json.loads(lag[0]['msg']) global_lag_count = msg['global_lag_count'] assert global_lag_count == (total_msg - 1)
def test_update_partitions_and_replica_factor_default_value(host): """ Check if can update partitions numbers make sure -1 values are considered, but warning + step is skipped """ # Given topic_name = get_topic_name() ensure_kafka_topic(host, topic_defaut_configuration, topic_name) time.sleep(0.3) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({'partitions': -1, 'replica_factor': -1}) ensure_kafka_topic(host, test_topic_configuration, topic_name) time.sleep(0.3) # Then expected_topic_configuration = topic_defaut_configuration.copy() expected_topic_configuration.update({'partitions': 1, 'replica_factor': 1}) for host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(host, expected_topic_configuration, topic_name, kfk_addr)
def test_rebalance_all_partitions_preserve_leader(host): """ Check if can reassign all topics-partition with leader conservation. """ # Given test_topic_name = get_topic_name() test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'partitions': 50, 'replica_factor': 2, 'preserve_leader': True }) ensure_kafka_topic(host, test_topic_configuration, test_topic_name) time.sleep(0.3) topics_config = call_kafka_info( host, { 'resource': 'topic-config', 'include_internal': True, 'include_defaults': False }) topics = call_kafka_info(host, { 'resource': 'topic', 'include_internal': True }) all_topics = topics[0]['ansible_module_results'] all_topics_config = topics_config[0]['ansible_module_results'] test_topics = [] for topic_name, partitions in all_topics.items(): if topic_name == test_topic_name or topic_name.startswith('__'): test_topics.append({ 'name': topic_name, 'partitions': len(partitions), 'replica_factor': len(partitions['0']['replicas']), 'options': all_topics_config[topic_name], 'force_reassign': True, 'preserve_leader': True, 'state': 'present' }) existing_leaders = [] for i, broker_topics in enumerate(topics): existing_leaders.append({}) for topic_name, partitions in ( broker_topics['ansible_module_results'].items()): if topic_name == test_topic_name or topic_name.startswith('__'): for partition, config in partitions.items(): existing_leaders[i][(topic_name, partition)] = \ config['leader'] # When test_topics_configuration = { 'topics': test_topics, 'kafka_sleep_time': 5, 'kafka_max_retries': 60 } ensure_kafka_topics(host, test_topics_configuration) time.sleep(10) # Then topics = call_kafka_info(host, { 'resource': 'topic', 'include_internal': True }) for i, broker_topics in enumerate(topics): resulting_replicas = {} for topic_name, partitions in ( topics[i]['ansible_module_results'].items()): if topic_name == test_topic_name or topic_name.startswith('__'): for partition, config in partitions.items(): resulting_replicas[(topic_name, partition)] = \ config['replicas'] for topic_partition, leader in existing_leaders[i].items(): assert leader in resulting_replicas[topic_partition] total_partitions = 0 partitions_per_broker = {} for topic_name, partitions in ( topics[i]['ansible_module_results'].items()): if topic_name == test_topic_name or topic_name.startswith('__'): for partition, config in partitions.items(): for replica in config['replicas']: if replica not in partitions_per_broker: partitions_per_broker[replica] = 0 partitions_per_broker[replica] += 1 total_partitions += 1 mean_partitions = int(total_partitions * 1.0 / len(partitions_per_broker)) for node_id, partitions in partitions_per_broker.items(): assert partitions >= mean_partitions, partitions_per_broker assert partitions <= (mean_partitions + 1), partitions_per_broker
def test_check_mode(host): """ Check if can check mode do nothing """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) test_acl_configuration = acl_defaut_configuration.copy() test_acl_configuration.update({ 'name': get_acl_name(), 'state': 'present', **sasl_default_configuration }) ensure_acl( host, test_acl_configuration ) time.sleep(0.3) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'state': 'absent' }) ensure_topic( host, test_topic_configuration, topic_name, check=True ) time.sleep(0.3) test_topic_configuration.update({ 'state': 'present', 'partitions': topic_defaut_configuration['partitions'] + 1, 'replica_factor': topic_defaut_configuration['replica_factor'] + 1, 'options': { 'retention.ms': 1000 } }) ensure_topic( host, test_topic_configuration, topic_name, check=True ) time.sleep(0.3) new_topic_name = get_topic_name() ensure_topic( host, test_topic_configuration, new_topic_name, check=True ) time.sleep(0.3) check_acl_configuration = test_acl_configuration.copy() check_acl_configuration.update({ 'state': 'absent' }) ensure_acl( host, check_acl_configuration, check=True ) time.sleep(0.3) check_acl_configuration.update({ 'state': 'present', 'name': get_topic_name() }) ensure_acl( host, check_acl_configuration, check=True ) time.sleep(0.3) # Then expected_topic_configuration = topic_defaut_configuration.copy() for kafka_host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] kfk_sasl_addr = "%s:9094" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(kafka_host, expected_topic_configuration, topic_name, kfk_addr) check_configured_acl(kafka_host, test_acl_configuration, kfk_sasl_addr) test_topic_configuration.update({ 'state': 'absent' }) check_configured_topic(kafka_host, test_topic_configuration, new_topic_name, kfk_addr) check_acl_configuration.update({ 'state': 'absent' }) check_configured_acl(kafka_host, test_acl_configuration, kfk_sasl_addr)