def test___ne___broker_list(self):
cluster_config1 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list=['kafka-cluster-1:9092', 'kafka-cluster-2:9092'],
zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,')
# Different broker list
cluster_config2 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list=['kafka-cluster-1:9092', 'kafka-cluster-3:9092'],
zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,')
assert cluster_config1 != cluster_config2
def test_get_all_clusters(self, mock_yaml):
topology = TopologyConfiguration(
cluster_type='mykafka',
kafka_topology_path=TEST_BASE_KAFKA,
)
actual_clusters = topology.get_all_clusters()
expected_clusters = [
ClusterConfig('mykafka', 'cluster1', ["mybrokerhost1:9092"],
"0.1.2.3,0.2.3.4/kafka"),
ClusterConfig('mykafka', 'cluster2', ["mybrokerhost2:9092"],
"0.3.4.5,0.4.5.6/kafka")
]
assert sorted(expected_clusters) == sorted(actual_clusters)
def cluster_config(self):
"""Returns a yelp_kafka.config.ClusterConfig.
This method will use :meth:`kafka_cluster_type` and
:meth:`kafka_cluster_name` to fetch a `ClusterConfig` using Yelp's
kafka discovery mechanism. If they both aren't specified, it will
fall back to creating a `ClusterConfig` from :meth:`kafka_broker_list`
and :meth:`kafka_zookeeper`. The default `ClusterConfig` will point
at the testing docker container.
"""
if (
self.kafka_cluster_type is not None and
self.kafka_cluster_name is not None and
not self.should_use_testing_containers
):
from yelp_kafka.discovery import get_kafka_cluster # NOQA
return get_kafka_cluster(self.kafka_cluster_type,
'data_pipeline-client',
self.kafka_cluster_name
)
else:
return ClusterConfig(
type='standard',
name='data_pipeline',
broker_list=self.kafka_broker_list,
zookeeper=self.kafka_zookeeper
)
def test___eq___broker_str(self):
cluster_config1 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list='kafka-cluster-1:9092,kafka-cluster-2:9092',
zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,')
cluster_config2 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list='kafka-cluster-1:9092,kafka-cluster-2:9092',
zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,')
assert cluster_config1 == cluster_config2
# Re-order the comma separated pair of brokers and zookeeper nodes
cluster_config2 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list='kafka-cluster-2:9092,kafka-cluster-1:9092',
zookeeper='zookeeper-cluster-2:2181,zookeeper-cluster-1:2181,')
assert cluster_config1 == cluster_config2
def test___eq___broker_list(self):
cluster_config1 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list=['kafka-cluster-1:9092', 'kafka-cluster-2:9092'],
zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,')
cluster_config2 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list=['kafka-cluster-1:9092', 'kafka-cluster-2:9092'],
zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,')
assert cluster_config1 == cluster_config2
# Re-ordering the list of brokers
cluster_config2 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list=['kafka-cluster-2:9092', 'kafka-cluster-1:9092'],
zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,')
assert cluster_config1 == cluster_config2
def test_get_cluster_by_name(self, mock_yaml):
topology = TopologyConfiguration(
cluster_type='mykafka',
kafka_topology_path=TEST_BASE_KAFKA,
)
actual_cluster = topology.get_cluster_by_name('cluster1')
expected_cluster = ClusterConfig('mykafka', 'cluster1',
["mybrokerhost1:9092"],
"0.1.2.3,0.2.3.4/kafka")
assert expected_cluster == actual_cluster
def test___ne___broker_str(self):
cluster_config1 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list='kafka-cluster-1:9092,kafka-cluster-2:9092',
zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,')
# Different comma separated pair of brokers
cluster_config2 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list='kafka-cluster-2:9092,kafka-cluster-3:9092',
zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,')
assert cluster_config1 != cluster_config2
# Different comma separated pair of zookeeper nodes
cluster_config2 = ClusterConfig(
type='some_type',
name='some_cluster',
broker_list='kafka-cluster-1:9092,kafka-cluster-2:9092',
zookeeper='zookeeper-cluster-2:2181,zookeeper-cluster-3:2181,')
assert cluster_config1 != cluster_config2
def test_get_local_cluster(self, mock_yaml):
topology = TopologyConfiguration(
cluster_type='mykafka',
kafka_topology_path=TEST_BASE_KAFKA,
)
mock_yaml.assert_called_once_with('/base/kafka_discovery/mykafka.yaml')
actual_cluster = topology.get_local_cluster()
expected_cluster = ClusterConfig(
'mykafka',
'cluster1',
['mybrokerhost1:9092'],
'0.1.2.3,0.2.3.4/kafka',
)
assert actual_cluster == expected_cluster
class TestMetadata(object):
cluster_config = ClusterConfig(
type='mytype',
name='some_cluster',
broker_list='some_list',
zookeeper='some_ip'
)
def test_get_topic_partition_metadata_empty(self, mock_get_metadata):
mock_get_metadata.return_value = {}
with mock.patch(
'kafka_utils.util.metadata.ZK',
):
actual = get_topic_partition_with_error(self.cluster_config, 5)
expected = set([])
assert actual == expected
mock_get_metadata.asserd_called_wity('some_list')
def test_get_topic_partition_metadata_no_errors(self, mock_get_metadata):
mock_get_metadata.return_value = METADATA_RESPONSE_ALL_FINE
with mock.patch(
'kafka_utils.util.metadata.ZK',
):
actual = get_topic_partition_with_error(self.cluster_config, 5)
expected = set([])
assert actual == expected
def test_get_topic_partition_metadata_replica_not_available(self, mock_get_metadata):
mock_get_metadata.return_value = METADATA_RESPONSE_WITH_ERRORS
with mock.patch(
'kafka_utils.util.metadata.ZK',
):
actual = get_topic_partition_with_error(self.cluster_config, REPLICA_NOT_AVAILABLE_ERROR)
expected = {('topic0', 0), ('topic1', 1)}
assert actual == expected
def test_get_topic_partition_metadata_leader_not_available(self, mock_get_metadata):
mock_get_metadata.return_value = METADATA_RESPONSE_WITH_ERRORS
with mock.patch(
'kafka_utils.util.metadata.ZK',
):
actual = get_topic_partition_with_error(self.cluster_config, LEADER_NOT_AVAILABLE_ERROR)
expected = set([('topic0', 1)])
assert actual == expected
class TestZK(object):
cluster_config = ClusterConfig(type='mytype',
name='some_cluster',
broker_list='some_list',
zookeeper='some_ip')
def test_create(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.create('/kafka/consumers/some_group/offsets')
zk.create('/kafka/consumers/some_group/offsets',
value='some_val',
acl=None,
ephemeral=True,
sequence=True,
makepath=True)
mock_obj = mock.Mock()
zk.create(
'/kafka/consumers/some_group/offsets',
value='some_val',
acl=mock_obj,
)
call_list = [
mock.call('/kafka/consumers/some_group/offsets', '', None,
False, False, False),
mock.call('/kafka/consumers/some_group/offsets', 'some_val',
None, True, True, True),
mock.call('/kafka/consumers/some_group/offsets', 'some_val',
mock_obj, False, False, False),
]
assert mock_client.return_value.create.call_args_list == call_list
def test_set(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.set('config/topics/some_topic', 'some_val')
zk.set('brokers/topics/some_topic',
'{"name": "some_topic", "more": "properties"}')
call_list = [
mock.call('config/topics/some_topic', 'some_val'),
mock.call('brokers/topics/some_topic',
'{"name": "some_topic", "more": "properties"}')
]
assert mock_client.return_value.set.call_args_list == call_list
def test_delete(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.delete('/kafka/consumers/some_group/offsets', )
zk.delete('/kafka/consumers/some_group/offsets', recursive=True)
call_list = [
mock.call('/kafka/consumers/some_group/offsets',
recursive=False),
mock.call('/kafka/consumers/some_group/offsets',
recursive=True),
]
assert mock_client.return_value.delete.call_args_list == call_list
def test_delete_topic(self, _):
with mock.patch.object(ZK, 'delete', autospec=True) as mock_delete:
with ZK(self.cluster_config) as zk:
zk.delete_topic(
'some_group',
'some_topic',
)
mock_delete.assert_called_once_with(
zk,
'/consumers/some_group/offsets/some_topic',
True,
)
def test_get_my_subscribed_partitions(self, _):
with mock.patch.object(
ZK,
'get_children',
autospec=True,
) as mock_children:
with ZK(self.cluster_config) as zk:
zk.get_my_subscribed_partitions(
'some_group',
'some_topic',
)
mock_children.assert_called_once_with(
zk,
'/consumers/some_group/offsets/some_topic',
)
def test_get_topic_config(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(return_value=(
b'{"version": 1, "config": {"cleanup.policy": "compact"}}',
"Random node info that doesn't matter"))
actual = zk.get_topic_config("some_topic")
expected = {"version": 1, "config": {"cleanup.policy": "compact"}}
assert actual == expected
def test_get_topic_config_8(self, mock_client):
"""
Test getting configuration for topics created in Kafa prior to 0.9.0.
"""
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(side_effect=NoNodeError())
zk.get_topics = mock.Mock(return_value={"some_topic": {}})
actual = zk.get_topic_config("some_topic")
expected = {"config": {}}
assert actual == expected
def test_get_nonexistent_topic_config(self, mock_client):
"""
Test getting configuration for topics that don't exist.
"""
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(side_effect=NoNodeError())
zk.get_topics = mock.Mock(return_value={})
with pytest.raises(NoNodeError):
zk.get_topic_config("some_topic")
def test_set_topic_config_kafka_10(self, mock_client):
with mock.patch.object(ZK, 'set', autospec=True) as mock_set:
with ZK(self.cluster_config) as zk:
config = {
"version": 1,
"config": {
"cleanup.policy": "compact"
}
}
config_change = {
"entity_path": "topics/some_topic",
"version": 2
}
zk.set_topic_config(
"some_topic",
config,
)
serialized_config = dump_json(config)
serialized_config_change = dump_json(config_change)
mock_set.assert_called_once_with(
zk,
'/config/topics/some_topic',
serialized_config,
)
expected_create_call = mock.call(
'/config/changes/config_change_', serialized_config_change,
None, False, True, False)
assert mock_client.return_value.create.call_args_list == [
expected_create_call
]
def test_set_topic_config_kafka_9(self, mock_client):
with mock.patch.object(ZK, 'set', autospec=True) as mock_set:
with ZK(self.cluster_config) as zk:
config = {
"version": 1,
"config": {
"cleanup.policy": "compact"
}
}
config_change = {
"version": 1,
"entity_type": "topics",
"entity_name": "some_topic"
}
zk.set_topic_config("some_topic", config, (0, 9, 2))
serialized_config = dump_json(config)
serialized_config_change = dump_json(config_change)
mock_set.assert_called_once_with(
zk,
'/config/topics/some_topic',
serialized_config,
)
expected_create_call = mock.call(
'/config/changes/config_change_', serialized_config_change,
None, False, True, False)
assert mock_client.return_value.create.call_args_list == [
expected_create_call
]
def test_get_broker_config(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(return_value=(
b'{"version": 1, "config": {"leader.replication.throttled.rate": "42"}}',
"Random node info that doesn't matter"))
actual = zk.get_broker_config(0)
expected = {
"version": 1,
"config": {
"leader.replication.throttled.rate": "42"
}
}
assert actual == expected
def test_set_broker_config_kafka_10(self, mock_client):
with mock.patch.object(ZK, 'set', autospec=True) as mock_set:
with ZK(self.cluster_config) as zk:
config = {
"version": 1,
"config": {
"leader.replication.throttled.rate": "42"
}
}
config_change = {"entity_path": "brokers/0", "version": 2}
zk.set_broker_config(0, config)
serialized_config = dump_json(config)
serialized_config_change = dump_json(config_change)
mock_set.assert_called_once_with(
zk,
'/config/brokers/0',
serialized_config,
)
expected_create_call = mock.call(
'/config/changes/config_change_', serialized_config_change,
None, False, True, False)
assert mock_client.return_value.create.call_args_list == [
expected_create_call
]
def test_get_topics(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(return_value=((
b'{"version": "1", "partitions": {"0": [1, 0]}}',
MockGetTopics(31000),
)))
zk._fetch_partition_state = mock.Mock(return_value=((
b'{"version": "2"}',
MockGetTopics(32000),
)))
actual_with_fetch_state = zk.get_topics("some_topic")
expected_with_fetch_state = {
'some_topic': {
'ctime': 31.0,
'partitions': {
'0': {
'replicas': [1, 0],
'ctime': 32.0,
'version': '2',
},
},
'version': '1',
},
}
assert actual_with_fetch_state == expected_with_fetch_state
zk._fetch_partition_info = mock.Mock(
return_value=MockGetTopics(33000))
actual_without_fetch_state = zk.get_topics(
"some_topic", fetch_partition_state=False)
expected_without_fetch_state = {
'some_topic': {
'ctime': 31.0,
'partitions': {
'0': {
'replicas': [1, 0],
'ctime': 33.0,
},
},
'version': '1',
},
}
assert actual_without_fetch_state == expected_without_fetch_state
def test_get_topics_empty_cluster(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.get_children = mock.Mock(side_effect=NoNodeError())
actual_with_no_node_error = zk.get_topics()
expected_with_no_node_error = {}
zk.get_children.assert_called_with("/brokers/topics")
assert actual_with_no_node_error == expected_with_no_node_error
def test_get_brokers_names_only(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.get_children = mock.Mock(return_value=[1, 2, 3], )
expected = {1: None, 2: None, 3: None}
actual = zk.get_brokers(names_only=True)
zk.get_children.assert_called_with("/brokers/ids")
assert actual == expected
def test_get_brokers_with_metadata(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.get_children = mock.Mock(return_value=[1, 2, 3], )
zk.get_broker_metadata = mock.Mock(return_value='broker', )
expected = {1: 'broker', 2: 'broker', 3: 'broker'}
actual = zk.get_brokers()
zk.get_children.assert_called_with("/brokers/ids")
calls = zk.get_broker_metadata.mock_calls
zk.get_broker_metadata.assert_has_calls(calls)
assert actual == expected
def test_get_brokers_empty_cluster(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.get_children = mock.Mock(side_effect=NoNodeError())
actual_with_no_node_error = zk.get_brokers()
expected_with_no_node_error = {}
zk.get_children.assert_called_with("/brokers/ids")
assert actual_with_no_node_error == expected_with_no_node_error
def test_get_brokers_with_metadata_for_ssl(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.get_children = mock.Mock(return_value=[1], )
zk.get = mock.Mock(return_value=(
b'{"endpoints":["SSL://broker:9093"],"host":null}', None))
expected = {1: {'host': 'broker'}}
actual = zk.get_brokers()
assert actual[1]['host'] == expected[1]['host']
zk.get = mock.Mock(return_value=(
b'{"endpoints":["INTERNAL://broker:9093","EXTERNAL://broker:9093"],"host":null}',
None))
expected = {1: {'host': 'broker'}}
actual = zk.get_brokers()
assert actual[1]['host'] == expected[1]['host']
def test_get_brokers_with_metadata_for_sasl(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.get_children = mock.Mock(return_value=[1], )
zk.get = mock.Mock(return_value=(
b'{"endpoints":["PLAINTEXTSASL://broker:9093"],"host":null}',
None))
expected = {1: {'host': 'broker'}}
actual = zk.get_brokers()
assert actual[1]['host'] == expected[1]['host']
def test_get_brokers_with_metadata_for_plaintext(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.get_children = mock.Mock(return_value=[1], )
zk.get = mock.Mock(
return_value=(b'{"endpoints":[],"host":"broker"}', None))
expected = {1: {'host': 'broker'}}
actual = zk.get_brokers()
assert actual[1]['host'] == expected[1]['host']
class TestZK(object):
cluster_config = ClusterConfig(type='mytype',
name='some_cluster',
broker_list='some_list',
zookeeper='some_ip')
def test_create(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.create('/kafka/consumers/some_group/offsets')
zk.create('/kafka/consumers/some_group/offsets',
value='some_val',
acl=None,
ephemeral=True,
sequence=True,
makepath=True)
mock_obj = mock.Mock()
zk.create(
'/kafka/consumers/some_group/offsets',
value='some_val',
acl=mock_obj,
)
call_list = [
mock.call('/kafka/consumers/some_group/offsets', '', None,
False, False, False),
mock.call('/kafka/consumers/some_group/offsets', 'some_val',
None, True, True, True),
mock.call('/kafka/consumers/some_group/offsets', 'some_val',
mock_obj, False, False, False),
]
assert mock_client.return_value.create.call_args_list == call_list
def test_set(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.set('config/topics/some_topic', 'some_val')
zk.set('brokers/topics/some_topic',
'{"name": "some_topic", "more": "properties"}')
call_list = [
mock.call('config/topics/some_topic', 'some_val'),
mock.call('brokers/topics/some_topic',
'{"name": "some_topic", "more": "properties"}')
]
assert mock_client.return_value.set.call_args_list == call_list
def test_delete(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.delete('/kafka/consumers/some_group/offsets', )
zk.delete('/kafka/consumers/some_group/offsets', recursive=True)
call_list = [
mock.call('/kafka/consumers/some_group/offsets',
recursive=False),
mock.call('/kafka/consumers/some_group/offsets',
recursive=True),
]
assert mock_client.return_value.delete.call_args_list == call_list
def test_delete_topic_partitions(self, mock_client):
with mock.patch.object(ZK, 'delete', autospec=True) as mock_delete:
with ZK(self.cluster_config) as zk:
zk.delete_topic_partitions('some_group', 'some_topic',
[0, 1, 2])
call_list = [
mock.call(zk,
'/consumers/some_group/offsets/some_topic/0'),
mock.call(zk,
'/consumers/some_group/offsets/some_topic/1'),
mock.call(zk,
'/consumers/some_group/offsets/some_topic/2'),
]
assert mock_delete.call_args_list == call_list
def test_delete_topic(self, _):
with mock.patch.object(ZK, 'delete', autospec=True) as mock_delete:
with ZK(self.cluster_config) as zk:
zk.delete_topic(
'some_group',
'some_topic',
)
mock_delete.assert_called_once_with(
zk,
'/consumers/some_group/offsets/some_topic',
True,
)
def test_get_my_subscribed_partitions(self, _):
with mock.patch.object(
ZK,
'get_children',
autospec=True,
) as mock_children:
with ZK(self.cluster_config) as zk:
zk.get_my_subscribed_partitions(
'some_group',
'some_topic',
)
mock_children.assert_called_once_with(
zk,
'/consumers/some_group/offsets/some_topic',
)
def test_get_topic_config(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(return_value=(
b'{"version": 1, "config": {"cleanup.policy": "compact"}}',
"Random node info that doesn't matter"))
actual = zk.get_topic_config("some_topic")
expected = {"version": 1, "config": {"cleanup.policy": "compact"}}
assert actual == expected
def test_set_topic_config_kafka_10(self, mock_client):
with mock.patch.object(ZK, 'set', autospec=True) as mock_set:
with ZK(self.cluster_config) as zk:
config = {
"version": 1,
"config": {
"cleanup.policy": "compact"
}
}
config_change = {
"entity_path": "topics/some_topic",
"version": 2
}
zk.set_topic_config(
"some_topic",
config,
)
serialized_config = dump_json(config)
serialized_config_change = dump_json(config_change)
mock_set.assert_called_once_with(
zk,
'/config/topics/some_topic',
serialized_config,
)
expected_create_call = mock.call(
'/config/changes/config_change_', serialized_config_change,
None, False, True, False)
assert mock_client.return_value.create.call_args_list == [
expected_create_call
]
def test_set_topic_config_kafka_9(self, mock_client):
with mock.patch.object(ZK, 'set', autospec=True) as mock_set:
with ZK(self.cluster_config) as zk:
config = {
"version": 1,
"config": {
"cleanup.policy": "compact"
}
}
config_change = {
"version": 1,
"entity_type": "topics",
"entity_name": "some_topic"
}
zk.set_topic_config("some_topic", config, (0, 9, 2))
serialized_config = dump_json(config)
serialized_config_change = dump_json(config_change)
mock_set.assert_called_once_with(
zk,
'/config/topics/some_topic',
serialized_config,
)
expected_create_call = mock.call(
'/config/changes/config_change_', serialized_config_change,
None, False, True, False)
assert mock_client.return_value.create.call_args_list == [
expected_create_call
]
main,
'read_cluster_status',
side_effect=read_cluster_state_values([], (100, 0)),
autospec=True,
)
@mock.patch('time.sleep', autospec=True)
def test_wait_for_stable_cluster_timeout(mock_sleep, mock_read):
with pytest.raises(main.WaitTimeoutException):
main.wait_for_stable_cluster([], 1, "", None, None, 5, 3, 100)
assert mock_read.call_count == 21
assert mock_sleep.mock_calls == [mock.call(5)] * 20
cluster_config = ClusterConfig(type='mytype',
name='some_cluster',
broker_list='some_list',
zookeeper='some_ip')
@mock.patch('kafka_utils.util.zookeeper.KazooClient', autospec=True)
@mock.patch.object(ZK,
'get_brokers',
side_effect=[{
2: {
'host': 'broker2'
},
3: {
'host': 'broker3'
},
1: {
'host': 'broker1'
class TestZK(object):
cluster_config = ClusterConfig(
type='mytype',
name='some_cluster',
broker_list='some_list',
zookeeper='some_ip'
)
def test_create(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.create(
'/kafka/consumers/some_group/offsets'
)
zk.create(
'/kafka/consumers/some_group/offsets',
value='some_val',
acl=None,
ephemeral=True,
sequence=True,
makepath=True
)
mock_obj = mock.Mock()
zk.create(
'/kafka/consumers/some_group/offsets',
value='some_val',
acl=mock_obj,
)
call_list = [
mock.call(
'/kafka/consumers/some_group/offsets',
'', None, False, False, False
),
mock.call(
'/kafka/consumers/some_group/offsets',
'some_val', None, True, True, True
),
mock.call(
'/kafka/consumers/some_group/offsets',
'some_val', mock_obj, False, False, False
),
]
assert mock_client.return_value.create.call_args_list == call_list
def test_set(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.set(
'config/topics/some_topic',
'some_val'
)
zk.set(
'brokers/topics/some_topic',
'{"name": "some_topic", "more": "properties"}'
)
call_list = [
mock.call(
'config/topics/some_topic',
'some_val'
),
mock.call(
'brokers/topics/some_topic',
'{"name": "some_topic", "more": "properties"}'
)
]
assert mock_client.return_value.set.call_args_list == call_list
def test_delete(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.delete(
'/kafka/consumers/some_group/offsets',
)
zk.delete(
'/kafka/consumers/some_group/offsets',
recursive=True
)
call_list = [
mock.call(
'/kafka/consumers/some_group/offsets',
recursive=False
),
mock.call(
'/kafka/consumers/some_group/offsets',
recursive=True
),
]
assert mock_client.return_value.delete.call_args_list == call_list
def test_delete_topic_partitions(self, mock_client):
with mock.patch.object(
ZK,
'delete',
autospec=True
) as mock_delete:
with ZK(self.cluster_config) as zk:
zk.delete_topic_partitions(
'some_group',
'some_topic',
[0, 1, 2]
)
call_list = [
mock.call(
zk,
'/consumers/some_group/offsets/some_topic/0'
),
mock.call(
zk,
'/consumers/some_group/offsets/some_topic/1'
),
mock.call(
zk,
'/consumers/some_group/offsets/some_topic/2'
),
]
assert mock_delete.call_args_list == call_list
def test_delete_topic(self, _):
with mock.patch.object(
ZK,
'delete',
autospec=True
) as mock_delete:
with ZK(self.cluster_config) as zk:
zk.delete_topic(
'some_group',
'some_topic',
)
mock_delete.assert_called_once_with(
zk,
'/consumers/some_group/offsets/some_topic',
True,
)
def test_get_my_subscribed_partitions(self, _):
with mock.patch.object(
ZK,
'get_children',
autospec=True,
) as mock_children:
with ZK(self.cluster_config) as zk:
zk.get_my_subscribed_partitions(
'some_group',
'some_topic',
)
mock_children.assert_called_once_with(
zk,
'/consumers/some_group/offsets/some_topic',
)
def test_get_topic_config(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(
return_value=(
b'{"version": 1, "config": {"cleanup.policy": "compact"}}',
"Random node info that doesn't matter"
)
)
actual = zk.get_topic_config("some_topic")
expected = {"version": 1, "config": {"cleanup.policy": "compact"}}
assert actual == expected
def test_get_topic_config_8(self, mock_client):
"""
Test getting configuration for topics created in Kafa prior to 0.9.0.
"""
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(side_effect=NoNodeError())
zk.get_topics = mock.Mock(return_value={"some_topic": {}})
actual = zk.get_topic_config("some_topic")
expected = {"config": {}}
assert actual == expected
def test_get_nonexistent_topic_config(self, mock_client):
"""
Test getting configuration for topics that don't exist.
"""
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(side_effect=NoNodeError())
zk.get_topics = mock.Mock(return_value={})
with pytest.raises(NoNodeError):
zk.get_topic_config("some_topic")
def test_set_topic_config_kafka_10(self, mock_client):
with mock.patch.object(
ZK,
'set',
autospec=True
) as mock_set:
with ZK(self.cluster_config) as zk:
config = {"version": 1, "config": {"cleanup.policy": "compact"}}
config_change = {"entity_path": "topics/some_topic", "version": 2}
zk.set_topic_config(
"some_topic",
config,
)
serialized_config = dump_json(config)
serialized_config_change = dump_json(config_change)
mock_set.assert_called_once_with(
zk,
'/config/topics/some_topic',
serialized_config,
)
expected_create_call = mock.call(
'/config/changes/config_change_',
serialized_config_change,
None,
False,
True,
False
)
assert mock_client.return_value.create.call_args_list == [expected_create_call]
def test_set_topic_config_kafka_9(self, mock_client):
with mock.patch.object(
ZK,
'set',
autospec=True
) as mock_set:
with ZK(self.cluster_config) as zk:
config = {"version": 1, "config": {"cleanup.policy": "compact"}}
config_change = {"version": 1, "entity_type": "topics", "entity_name": "some_topic"}
zk.set_topic_config(
"some_topic",
config,
(0, 9, 2)
)
serialized_config = dump_json(config)
serialized_config_change = dump_json(config_change)
mock_set.assert_called_once_with(
zk,
'/config/topics/some_topic',
serialized_config,
)
expected_create_call = mock.call(
'/config/changes/config_change_',
serialized_config_change,
None,
False,
True,
False
)
assert mock_client.return_value.create.call_args_list == [expected_create_call]
def test_get_topics(self, mock_client):
with ZK(self.cluster_config) as zk:
zk.zk.get = mock.Mock(
return_value=(
(
b'{"version": "1", "partitions": {"0": [1, 0]}}',
TestGetTopics(31000),
)
)
)
zk._fetch_partition_state = mock.Mock(
return_value=(
(
b'{"version": "2"}',
TestGetTopics(32000),
)
)
)
actual_with_fetch_state = zk.get_topics("some_topic")
expected_with_fetch_state = {
'some_topic': {
'ctime': 31.0,
'partitions': {
'0': {
'replicas': [1, 0],
'ctime': 32.0,
'version': '2',
},
},
'version': '1',
},
}
assert actual_with_fetch_state == expected_with_fetch_state
zk._fetch_partition_info = mock.Mock(
return_value=TestGetTopics(33000)
)
actual_without_fetch_state = zk.get_topics("some_topic", fetch_partition_state=False)
expected_without_fetch_state = {
'some_topic': {
'ctime': 31.0,
'partitions': {
'0': {
'replicas': [1, 0],
'ctime': 33.0,
},
},
'version': '1',
},
}
assert actual_without_fetch_state == expected_without_fetch_state