def test_update_partitions(host): """ Check if can update partitions numbers """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'partitions': 2 }) ensure_idempotency( ensure_topic, host, test_topic_configuration, topic_name ) time.sleep(0.3) # Then for kafka_host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(kafka_host, test_topic_configuration, topic_name, kfk_addr)
def test_kafka_info_topics_config_not_include_defaults(host): """ Check if can get config on topic. """ # Given topic_name = get_topic_name() test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'options': { 'retention.ms': 66574936 } }) ensure_topic( host, test_topic_configuration, topic_name ) time.sleep(0.3) # When results = call_kafka_info( host, { 'resource': 'topic-config', 'include_defaults': False }, ) # Then for r in results: assert topic_name in r['ansible_module_results'] for name, topic_config in r['ansible_module_results'].items(): if name == topic_name: assert int(topic_config['retention.ms']) == 66574936 assert 'min.insync.replicas' not in topic_config
def test_kafka_info_consumer_group(host): """ Check if can get info on consumer groups """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) consumer_group = get_consumer_group() total_msg = 42 # When produce_and_consume_topic( topic_name, total_msg, consumer_group) time.sleep(0.3) # Then results = call_kafka_info( host, { 'resource': 'consumer_group' } ) for r in results: assert consumer_group in r['ansible_module_results']
def test_consumer_lag(host): """ Check if can check global consumer lag """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) consumer_group = get_consumer_group() total_msg = 42 # When produce_and_consume_topic( topic_name, total_msg, consumer_group) time.sleep(0.3) # Then lags = call_kafka_stat_lag(host, { 'consummer_group': consumer_group }) for lag in lags: msg = json.loads(lag['msg']) global_lag_count = msg['global_lag_count'] assert global_lag_count == (total_msg - 1)
def test_delete_options(): """ Check if can remove topic options """ # Given init_topic_configuration = topic_defaut_configuration.copy() init_topic_configuration.update( {'options': { 'retention.ms': 66574936, 'flush.ms': 564939 }}) topic_name = get_topic_name() ensure_topic(localhost, init_topic_configuration, topic_name) time.sleep(0.5) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({'options': {'flush.ms': 564939}}) ensure_idempotency(ensure_topic, localhost, test_topic_configuration, topic_name) time.sleep(0.5) # Then deleted_options = { 'retention.ms': 66574936, } for host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(host, test_topic_configuration, topic_name, kfk_addr, deleted_options=deleted_options)
def test_add_options(host): """ Check if can update topic options """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'options': { 'retention.ms': 66574936, 'flush.ms': 564939 } }) ensure_idempotency( ensure_topic, host, test_topic_configuration, topic_name ) time.sleep(0.3) # Then for kafka_host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(kafka_host, test_topic_configuration, topic_name, kfk_addr)
def test_kafka_info_topic(host): """ Check if can get info on topic """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) topic_test_name = get_topic_name() topic_test_configuration = topic_defaut_configuration.copy() topic_test_configuration.update({ 'options': { 'min.insync.replicas': 2 } }) ensure_topic( host, topic_test_configuration, topic_test_name ) time.sleep(0.3) produce_and_consume_topic(topic_name, 10, get_consumer_group()) time.sleep(0.3) # When results = call_kafka_info( host, { 'resource': 'topic' } ) # Then for r in results: assert topic_name in r['ansible_module_results'] assert '__consumer_offsets' not in r['ansible_module_results'] for name, topic_info in r['ansible_module_results'].items(): for partition, partition_info in topic_info.items(): assert 'earliest_offset' in partition_info assert 'latest_offset' in partition_info assert partition_info['earliest_offset'] >= 0 assert (partition_info['latest_offset'] == 10 if name == topic_name else partition_info['latest_offset'] >= 0) if name == topic_name: assert partition_info['at_min_isr'] assert not partition_info['under_replicated'] assert not partition_info['under_min_isr'] assert not partition_info['unavailable_partition'] elif name == topic_test_name: assert not partition_info['at_min_isr'] assert not partition_info['under_replicated'] assert partition_info['under_min_isr'] assert not partition_info['unavailable_partition']
def test_kafka_info_brokers(): """ Check if can get info on brokers """ # Given topic_name = get_topic_name() ensure_topic(localhost, topic_defaut_configuration, topic_name) time.sleep(0.5) # When results = call_kafka_info(localhost, {'resource': 'broker'}) # Then for r in results: assert len(r['ansible_module_results']) == 2
def test_kafka_info_topic(host): """ Check if can get info on topic """ # Given topic_name = get_topic_name() ensure_topic(host, topic_defaut_configuration, topic_name) time.sleep(0.3) # When results = call_kafka_info(host, {'resource': 'topic'}) # Then for r in results: assert topic_name in r['ansible_module_results']
def test_update_replica_factor(): """ Check if can update replication factor """ # Given topic_name = get_topic_name() ensure_topic(localhost, topic_defaut_configuration, topic_name) time.sleep(0.5) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({'replica_factor': 2}) ensure_idempotency(ensure_topic, localhost, test_topic_configuration, topic_name) time.sleep(0.5) # Then for host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(host, test_topic_configuration, topic_name, kfk_addr)
def test_consumer_lag(): """ Check if can check global consumer lag """ # Given topic_name = get_topic_name() ensure_topic(localhost, topic_defaut_configuration, topic_name) time.sleep(0.5) consumer_group = get_consumer_group() total_msg = 42 # When produce_and_consume_topic(topic_name, total_msg, consumer_group) time.sleep(0.5) # Then for host, host_vars in kafka_hosts.items(): lag = call_kafka_stat_lag(localhost, {'consummer_group': consumer_group}) msg = json.loads(lag[0]['msg']) global_lag_count = msg['global_lag_count'] assert global_lag_count == (total_msg - 1)
def test_kafka_info_topic_include_internal(host): """ Check if can get info on topic """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) topic_test_name = get_topic_name() topic_test_configuration = topic_defaut_configuration.copy() topic_test_configuration.update({ 'options': { 'min.insync.replicas': 2 } }) ensure_topic( host, topic_test_configuration, topic_test_name ) time.sleep(0.3) produce_and_consume_topic(topic_name, 10, get_consumer_group()) time.sleep(0.3) # When results = call_kafka_info( host, { 'resource': 'topic', 'include_internal': True } ) # Then for r in results: assert topic_name in r['ansible_module_results'] assert '__consumer_offsets' in r['ansible_module_results']
def test_update_partitions_and_replica_factor_default_value(): """ Check if can update partitions numbers make sure -1 values are considered, but warning + step is skipped """ # Given topic_name = get_topic_name() ensure_topic(localhost, topic_defaut_configuration, topic_name) time.sleep(0.5) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({'partitions': -1, 'replica_factor': -1}) ensure_topic(localhost, test_topic_configuration, topic_name) time.sleep(0.5) # Then expected_topic_configuration = topic_defaut_configuration.copy() expected_topic_configuration.update({'partitions': 1, 'replica_factor': 1}) for host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(host, expected_topic_configuration, topic_name, kfk_addr)
def test_check_mode(host): """ Check if can check mode do nothing """ # Given topic_name = get_topic_name() ensure_topic( host, topic_defaut_configuration, topic_name ) time.sleep(0.3) test_acl_configuration = acl_defaut_configuration.copy() test_acl_configuration.update({ 'name': get_acl_name(), 'state': 'present', **sasl_default_configuration }) ensure_acl( host, test_acl_configuration ) time.sleep(0.3) # When test_topic_configuration = topic_defaut_configuration.copy() test_topic_configuration.update({ 'state': 'absent' }) ensure_topic( host, test_topic_configuration, topic_name, check=True ) time.sleep(0.3) test_topic_configuration.update({ 'state': 'present', 'partitions': topic_defaut_configuration['partitions'] + 1, 'replica_factor': topic_defaut_configuration['replica_factor'] + 1, 'options': { 'retention.ms': 1000 } }) ensure_topic( host, test_topic_configuration, topic_name, check=True ) time.sleep(0.3) new_topic_name = get_topic_name() ensure_topic( host, test_topic_configuration, new_topic_name, check=True ) time.sleep(0.3) check_acl_configuration = test_acl_configuration.copy() check_acl_configuration.update({ 'state': 'absent' }) ensure_acl( host, check_acl_configuration, check=True ) time.sleep(0.3) check_acl_configuration.update({ 'state': 'present', 'name': get_topic_name() }) ensure_acl( host, check_acl_configuration, check=True ) time.sleep(0.3) # Then expected_topic_configuration = topic_defaut_configuration.copy() for kafka_host, host_vars in kafka_hosts.items(): kfk_addr = "%s:9092" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] kfk_sasl_addr = "%s:9094" % \ host_vars['ansible_eth0']['ipv4']['address']['__ansible_unsafe'] check_configured_topic(kafka_host, expected_topic_configuration, topic_name, kfk_addr) check_configured_acl(kafka_host, test_acl_configuration, kfk_sasl_addr) test_topic_configuration.update({ 'state': 'absent' }) check_configured_topic(kafka_host, test_topic_configuration, new_topic_name, kfk_addr) check_acl_configuration.update({ 'state': 'absent' }) check_configured_acl(kafka_host, test_acl_configuration, kfk_sasl_addr)