def test_reducing_replication(self, kazoo_client, subprocess, tmpdir): tmp_file = tmpdir.mkdir('sub').join( 'kafka_topic_partition_realocation.json') existing = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '2', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, 'partitions': [ { 'topic': 'test', 'partition': '0', 'replicas': ['1', '2'] }, { 'topic': 'test', 'partition': '1', 'replicas': ['3', '4'] }, ] } } new = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '1', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, } } subprocess.Popen.side_effect = self.popen_valid with mock.patch( 'enforcer.kafka_topic_enforcer.file_kafka_reassignment', new=tmp_file.strpath), \ mock.patch('enforcer.kafka_topic_enforcer.wait_for_relocation', new=2): kafka_topic_enforcer.add_replicas(existing, new) with open(tmp_file.strpath, 'r') as f: config = json.load(f) assert config == { 'version': 1, 'partitions': [ {'topic': 'test', 'partition': 0, 'replicas': [1]}, {'topic': 'test', 'partition': 1, 'replicas': [3]} ] } subprocess.Popen.assert_any_call( ['kafka/home/bin/kafka-reassign-partitions.sh', '--zookeeper', 'localhost:2181', '--reassignment-json-file', tmp_file.strpath, '--execute'], stdout=subprocess.PIPE)
def test_no_change(self, kazoo_client, subprocess): existing = { 'test': { 'PartitionCount': '1', 'ReplicationFactor': '4', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, } } new = { 'test': { 'PartitionCount': '1', 'ReplicationFactor': '4', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, } } kafka_topic_enforcer.add_replicas(existing, new) assert not subprocess.Popen.called
def test_not_changing_hosts(self, kazoo_client, subprocess, tmpdir): tmp_file = tmpdir.mkdir('sub').join( 'kafka_topic_partition_realocation.json') kazoo_client.return_value.get_children.return_value = ['1', '2', '3', '4', '5'] # all # the nodes existing = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '2', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, 'partitions': [ { 'topic': 'test', 'partition': '0', 'replicas': ['3', '4'] }, { 'topic': 'test', 'partition': '1', 'replicas': ['3', '4'] }, ] } } new = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '2', 'hosts': ['3', '4'], 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, } } subprocess.Popen.side_effect = self.popen_valid kafka_topic_enforcer.add_replicas(existing, new) assert not subprocess.Popen.called
def test_increasing_replication_too_high(self, kazoo_client, subprocess): kazoo_client.return_value.get_children.return_value = ['1', '2', '3', '4', '5'] # all # the nodes existing = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '2', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, 'partitions': [ { 'topic': 'test', 'partition': '0', 'replicas': ['1', '2'] }, { 'topic': 'test', 'partition': '1', 'replicas': ['3', '4'] }, ] } } new = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '6', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, } } subprocess.Popen.side_effect = self.popen_valid with pytest.raises(Exception): kafka_topic_enforcer.add_replicas(existing, new) assert not subprocess.Popen.called
def test_reducing_replication_too_low(self, kazoo_client, subprocess): existing = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '2', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, 'partitions': [ { 'topic': 'test', 'partition': '0', 'replicas': ['1', '2'] }, { 'topic': 'test', 'partition': '1', 'replicas': ['3', '4'] }, ] } } new = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '0', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, } } subprocess.Popen.side_effect = self.popen_valid with pytest.raises(Exception): kafka_topic_enforcer.add_replicas(existing, new) assert not subprocess.Popen.called
def test_increasing_replication(self, kazoo_client, subprocess, tmpdir): tmp_file = tmpdir.mkdir('sub').join( 'kafka_topic_partition_realocation.json') kazoo_client.return_value.get_children.return_value = ['1', '2', '3', '4', '5'] # all # the nodes existing = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '2', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, 'partitions': [ { 'topic': 'test', 'partition': '0', 'replicas': ['1', '2'] }, { 'topic': 'test', 'partition': '1', 'replicas': ['3', '4'] }, ] } } new = { 'test': { 'PartitionCount': '2', 'ReplicationFactor': '3', 'Topic': 'test', 'Configs': {'retention.ms': '8640000'}, } } subprocess.Popen.side_effect = self.popen_valid with mock.patch( 'enforcer.kafka_topic_enforcer.file_kafka_reassignment', new=tmp_file.strpath), \ mock.patch('enforcer.kafka_topic_enforcer.wait_for_relocation', new=2): kafka_topic_enforcer.add_replicas(existing, new) with open(tmp_file.strpath, 'r') as f: config = json.load(f) assert len(config['partitions']) == 2 assert config['partitions'][0]['replicas'] in [[1, 2, 3], [1, 2, 4], [1, 2, 5 ]] assert len(config['partitions'][0]['replicas']) == 3 assert config['partitions'][1]['replicas'] in [[3, 4, 1], [3, 4, 2], [3, 4, 5 ]] assert len(config['partitions'][1]['replicas']) == 3 subprocess.Popen.assert_any_call( ['kafka/home/bin/kafka-reassign-partitions.sh', '--zookeeper', 'localhost:2181', '--reassignment-json-file', tmp_file.strpath, '--execute'], stdout=subprocess.PIPE)