def test_reducing_replication(self, kazoo_client, subprocess, tmpdir):
tmp_file = tmpdir.mkdir('sub').join(
'kafka_topic_partition_realocation.json')
existing = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '2',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
'partitions': [
{
'topic': 'test',
'partition': '0',
'replicas': ['1', '2']
},
{
'topic': 'test',
'partition': '1',
'replicas': ['3', '4']
},
]
}
}
new = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '1',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
}
}
subprocess.Popen.side_effect = self.popen_valid
with mock.patch(
'enforcer.kafka_topic_enforcer.file_kafka_reassignment',
new=tmp_file.strpath), \
mock.patch('enforcer.kafka_topic_enforcer.wait_for_relocation',
new=2):
kafka_topic_enforcer.add_replicas(existing, new)
with open(tmp_file.strpath, 'r') as f:
config = json.load(f)
assert config == {
'version': 1,
'partitions': [
{'topic': 'test',
'partition': 0,
'replicas': [1]},
{'topic': 'test',
'partition': 1,
'replicas': [3]}
]
}
subprocess.Popen.assert_any_call(
['kafka/home/bin/kafka-reassign-partitions.sh',
'--zookeeper', 'localhost:2181', '--reassignment-json-file',
tmp_file.strpath, '--execute'],
stdout=subprocess.PIPE)
def test_no_change(self, kazoo_client, subprocess):
existing = {
'test': {
'PartitionCount': '1',
'ReplicationFactor': '4',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
}
}
new = {
'test': {
'PartitionCount': '1',
'ReplicationFactor': '4',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
}
}
kafka_topic_enforcer.add_replicas(existing, new)
assert not subprocess.Popen.called
def test_not_changing_hosts(self, kazoo_client, subprocess, tmpdir):
tmp_file = tmpdir.mkdir('sub').join(
'kafka_topic_partition_realocation.json')
kazoo_client.return_value.get_children.return_value = ['1', '2', '3',
'4', '5'] # all
# the nodes
existing = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '2',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
'partitions': [
{
'topic': 'test',
'partition': '0',
'replicas': ['3', '4']
},
{
'topic': 'test',
'partition': '1',
'replicas': ['3', '4']
},
]
}
}
new = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '2',
'hosts': ['3', '4'],
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
}
}
subprocess.Popen.side_effect = self.popen_valid
kafka_topic_enforcer.add_replicas(existing, new)
assert not subprocess.Popen.called
def test_increasing_replication_too_high(self, kazoo_client, subprocess):
kazoo_client.return_value.get_children.return_value = ['1', '2', '3',
'4', '5'] # all
# the nodes
existing = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '2',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
'partitions': [
{
'topic': 'test',
'partition': '0',
'replicas': ['1', '2']
},
{
'topic': 'test',
'partition': '1',
'replicas': ['3', '4']
},
]
}
}
new = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '6',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
}
}
subprocess.Popen.side_effect = self.popen_valid
with pytest.raises(Exception):
kafka_topic_enforcer.add_replicas(existing, new)
assert not subprocess.Popen.called
def test_reducing_replication_too_low(self, kazoo_client, subprocess):
existing = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '2',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
'partitions': [
{
'topic': 'test',
'partition': '0',
'replicas': ['1', '2']
},
{
'topic': 'test',
'partition': '1',
'replicas': ['3', '4']
},
]
}
}
new = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '0',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
}
}
subprocess.Popen.side_effect = self.popen_valid
with pytest.raises(Exception):
kafka_topic_enforcer.add_replicas(existing, new)
assert not subprocess.Popen.called
def test_increasing_replication(self, kazoo_client, subprocess, tmpdir):
tmp_file = tmpdir.mkdir('sub').join(
'kafka_topic_partition_realocation.json')
kazoo_client.return_value.get_children.return_value = ['1', '2', '3',
'4', '5'] # all
# the nodes
existing = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '2',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
'partitions': [
{
'topic': 'test',
'partition': '0',
'replicas': ['1', '2']
},
{
'topic': 'test',
'partition': '1',
'replicas': ['3', '4']
},
]
}
}
new = {
'test': {
'PartitionCount': '2',
'ReplicationFactor': '3',
'Topic': 'test',
'Configs': {'retention.ms': '8640000'},
}
}
subprocess.Popen.side_effect = self.popen_valid
with mock.patch(
'enforcer.kafka_topic_enforcer.file_kafka_reassignment',
new=tmp_file.strpath), \
mock.patch('enforcer.kafka_topic_enforcer.wait_for_relocation',
new=2):
kafka_topic_enforcer.add_replicas(existing, new)
with open(tmp_file.strpath, 'r') as f:
config = json.load(f)
assert len(config['partitions']) == 2
assert config['partitions'][0]['replicas'] in [[1, 2, 3],
[1, 2, 4], [1, 2, 5
]]
assert len(config['partitions'][0]['replicas']) == 3
assert config['partitions'][1]['replicas'] in [[3, 4, 1],
[3, 4, 2], [3, 4, 5
]]
assert len(config['partitions'][1]['replicas']) == 3
subprocess.Popen.assert_any_call(
['kafka/home/bin/kafka-reassign-partitions.sh',
'--zookeeper', 'localhost:2181', '--reassignment-json-file',
tmp_file.strpath, '--execute'],
stdout=subprocess.PIPE)
