def write_messages(): # kafka may not be ready to accept all msgs, try till all are done def fn(num): try: offset_info = get_kafka_command( 'topic offsets {}'.format(TOPIC_NAME)) offset = int(offset_info[0]['0']) if offset < num: get_kafka_command('topic producer_test {} {}'.format( TOPIC_NAME, num - offset)) assert (num - offset) >= 0 return num - offset except RuntimeError: return num def success_predicate(left_offset): return (left_offset <= 0, 'producer_test continues....') get_kafka_command('topic producer_test {} {}'.format( TOPIC_NAME, NUM_TEST_MSGS)) spin(fn, success_predicate, NUM_TEST_MSGS) print('producer_test is successful {} msg available'.format(NUM_TEST_MSGS)) # double check check_offsets()
def test_topic_offsets_increase_with_writes(): offset_info = get_kafka_command( 'topic offsets --time="-1" {}'.format(DEFAULT_TOPIC_NAME) ) assert len(offset_info) == DEFAULT_PARTITION_COUNT offsets = {} for o in offset_info: assert len(o) == DEFAULT_REPLICATION_FACTOR offsets.update(o) assert len(offsets) == DEFAULT_PARTITION_COUNT num_messages = 10 write_info = get_kafka_command( 'topic producer_test {} {}'.format(DEFAULT_TOPIC_NAME, num_messages) ) assert len(write_info) == 1 assert write_info['message'].startswith( 'Output: {} records sent'.format(num_messages) ) offset_info = get_kafka_command( 'topic offsets --time="-1" {}'.format(DEFAULT_TOPIC_NAME) ) assert len(offset_info) == DEFAULT_PARTITION_COUNT post_write_offsets = {} for offsets in offset_info: assert len(o) == DEFAULT_REPLICATION_FACTOR post_write_offsets.update(o) assert not offsets == post_write_offsets
def test_topic_offsets_increase_with_writes(): offset_info = get_kafka_command( 'topic offsets --time="-1" {}'.format(DEFAULT_TOPIC_NAME)) assert len(offset_info) == DEFAULT_PARTITION_COUNT offsets = {} for o in offset_info: assert len(o) == DEFAULT_REPLICATION_FACTOR offsets.update(o) assert len(offsets) == DEFAULT_PARTITION_COUNT num_messages = 10 write_info = get_kafka_command('topic producer_test {} {}'.format( DEFAULT_TOPIC_NAME, num_messages)) assert len(write_info) == 1 assert write_info['message'].startswith( 'Output: {} records sent'.format(num_messages)) offset_info = get_kafka_command( 'topic offsets --time="-1" {}'.format(DEFAULT_TOPIC_NAME)) assert len(offset_info) == DEFAULT_PARTITION_COUNT post_write_offsets = {} for offsets in offset_info: assert len(o) == DEFAULT_REPLICATION_FACTOR post_write_offsets.update(o) assert not offsets == post_write_offsets
def test_config(): configs = get_kafka_command('config list') assert len(configs) == 1 assert get_kafka_command('config show {}'.format(configs[0])) assert get_kafka_command('config target') assert get_kafka_command('config target_id')
def fn(num): try: offset_info = get_kafka_command( 'topic offsets {}'.format(TOPIC_NAME)) offset = int(offset_info[0]['0']) if offset < num: get_kafka_command('topic producer_test {} {}'.format( TOPIC_NAME, num - offset)) assert (num - offset) >= 0 return num - offset except RuntimeError: return num
def test_topic_deletion_succeeds(): delete_info = get_kafka_command( 'topic delete {}'.format(EPHEMERAL_TOPIC_NAME)) assert len(delete_info) == 1 assert delete_info['message'].startswith( 'Output: Topic {} is marked for deletion'.format(EPHEMERAL_TOPIC_NAME)) # Make sure the topic isn't _actually_ deleted yet. topic_info = get_kafka_command( 'topic describe {}'.format(EPHEMERAL_TOPIC_NAME)) assert len(topic_info) == 1 assert len(topic_info['partitions']) == DEFAULT_PARTITION_COUNT
def test_no_under_replicated_topics_exist(): partition_info = get_kafka_command( 'topic under_replicated_partitions' ) assert len(partition_info) == 1 assert partition_info['message'] == ''
def test_topic_creation_succeeds(): create_info = get_kafka_command( 'topic create {}'.format(EPHEMERAL_TOPIC_NAME)) assert ('Created topic "%s".\n' % EPHEMERAL_TOPIC_NAME in create_info['message']) assert ("topics with a period ('.') or underscore ('_') could collide." in create_info['message']) topic_list_info = get_kafka_command('topic list') assert topic_list_info == [EPHEMERAL_TOPIC_NAME] topic_info = get_kafka_command( 'topic describe {}'.format(EPHEMERAL_TOPIC_NAME)) assert len(topic_info) == 1 assert len(topic_info['partitions']) == DEFAULT_PARTITION_COUNT
def test_single_broker_replace_succeeds(): broker_0_task = get_running_broker_task('broker-0')[0] broker_0_id = broker_0_task['id'] assert broker_0_id.startswith('broker-0__') replace_info = get_kafka_command('broker replace 0') task_id_changes('broker-0', broker_0_id)
def test_topic_deletion_succeeds(): delete_info = get_kafka_command( 'topic delete {}'.format(EPHEMERAL_TOPIC_NAME) ) assert len(delete_info) == 1 assert delete_info['message'].startswith( 'Output: Topic {} is marked for deletion'.format(EPHEMERAL_TOPIC_NAME) ) # Make sure the topic isn't _actually_ deleted yet. topic_info = get_kafka_command( 'topic describe {}'.format(EPHEMERAL_TOPIC_NAME) ) assert len(topic_info) == 1 assert len(topic_info['partitions']) == DEFAULT_PARTITION_COUNT
def test_single_broker_restart_succeeds(): for i in range(DEFAULT_BROKER_COUNT): restart_info = get_kafka_command('broker restart {}'.format(i)) assert len(restart_info) == 1 assert restart_info[0].startswith('broker-{}__'.format(i)) get_broker_list()
def test_restart_all_brokers_succeeds(): for i in range(DEFAULT_BROKER_COUNT): broker_task = get_running_broker_task('broker-{}'.format(i))[0] broker_id = broker_task['id'] assert broker_id.startswith('broker-{}__'.format(i)) restart_info = get_kafka_command('broker restart {}'.format(i)) task_id_changes('broker-{}'.format(i), broker_id) assert len(restart_info) == 1 assert restart_info[0].startswith('broker-{}__'.format(i))
def fn(): try: offset_info = get_kafka_command( 'topic offsets {}'.format(TOPIC_NAME)) offset = int(offset_info[0]['0']) assert offset == NUM_TEST_MSGS return True except RuntimeError: return False
def test_increasing_topic_partitions_succeeds(): partition_info = get_kafka_command('topic partitions {} {}'.format( DEFAULT_TOPIC_NAME, DEFAULT_PARTITION_COUNT + 1)) assert len(partition_info) == 1 assert partition_info['message'].startswith( 'Output: WARNING: If partitions are increased') assert ('The number of partitions for a topic can only be increased' not in partition_info['message'])
def test_topic_creation_succeeds(): create_info = get_kafka_command( 'topic create {}'.format(EPHEMERAL_TOPIC_NAME) ) assert ( 'Created topic "%s".\n' % EPHEMERAL_TOPIC_NAME in create_info['message'] ) assert ( "topics with a period ('.') or underscore ('_') could collide." in create_info['message'] ) topic_list_info = get_kafka_command('topic list') assert topic_list_info == [EPHEMERAL_TOPIC_NAME] topic_info = get_kafka_command( 'topic describe {}'.format(EPHEMERAL_TOPIC_NAME) ) assert len(topic_info) == 1 assert len(topic_info['partitions']) == DEFAULT_PARTITION_COUNT
def test_increasing_topic_partitions_succeeds(): partition_info = get_kafka_command( 'topic partitions {} {}'.format( DEFAULT_TOPIC_NAME, DEFAULT_PARTITION_COUNT + 1 ) ) assert len(partition_info) == 1 assert partition_info['message'].startswith( 'Output: WARNING: If partitions are increased' ) assert ( 'The number of partitions for a topic can only be increased' not in partition_info['message'] )
def test_topic_partition_count_is_correct(default_topic): topic_info = get_kafka_command( 'topic describe {}'.format(DEFAULT_TOPIC_NAME)) assert len(topic_info['partitions']) == DEFAULT_PARTITION_COUNT
def test_topic_partition_count_is_correct(default_topic): topic_info = get_kafka_command( 'topic describe {}'.format(DEFAULT_TOPIC_NAME) ) assert len(topic_info['partitions']) == DEFAULT_PARTITION_COUNT
def fn(): return get_kafka_command('connection')
def fn(): try: get_kafka_command('topic list') return True except RuntimeError: return False
def fn(): try: return get_kafka_command('broker list') except RuntimeError: return []
def test_expected_brokers_exist(): broker_info = get_kafka_command('broker list') assert set(broker_info) == set([str(i) for i in range(3)])
def test_no_unavailable_partitions_exist(): partition_info = get_kafka_command('topic unavailable_partitions') assert len(partition_info) == 1 assert partition_info['message'] == ''
def test_invalid_broker_id_returns_null(): restart_info = get_kafka_command( 'broker restart {}'.format(DEFAULT_BROKER_COUNT + 1)) assert len(restart_info) == 1 assert restart_info[0] is None
def test_state(): assert get_kafka_command('state framework_id') tasks = get_kafka_command('state tasks') assert tasks
def test_help(): get_kafka_command('help')
def check_offsets(): offset_info = get_kafka_command('topic offsets {}'.format(TOPIC_NAME)) offset = int(offset_info[0]['0']) assert offset == NUM_TEST_MSGS
def write_messages(): get_kafka_command('topic producer_test {} {}'.format( TOPIC_NAME, NUM_TEST_MSGS)) check_offsets()
def test_plan(): assert get_kafka_command('plan') assert get_kafka_command('continue') assert get_kafka_command('interrupt')
def test_invalid_broker_id_returns_null(): restart_info = get_kafka_command('broker restart {}'.format(DEFAULT_BROKER_COUNT + 1)) assert len(restart_info) == 1 assert restart_info[0] is None