def test___ne___broker_list(self): cluster_config1 = ClusterConfig( type='some_type', name='some_cluster', broker_list=['kafka-cluster-1:9092', 'kafka-cluster-2:9092'], zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,') # Different broker list cluster_config2 = ClusterConfig( type='some_type', name='some_cluster', broker_list=['kafka-cluster-1:9092', 'kafka-cluster-3:9092'], zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,') assert cluster_config1 != cluster_config2
def test_get_all_clusters(self, mock_yaml): topology = TopologyConfiguration( cluster_type='mykafka', kafka_topology_path=TEST_BASE_KAFKA, ) actual_clusters = topology.get_all_clusters() expected_clusters = [ ClusterConfig('mykafka', 'cluster1', ["mybrokerhost1:9092"], "0.1.2.3,0.2.3.4/kafka"), ClusterConfig('mykafka', 'cluster2', ["mybrokerhost2:9092"], "0.3.4.5,0.4.5.6/kafka") ] assert sorted(expected_clusters) == sorted(actual_clusters)
def cluster_config(self): """Returns a yelp_kafka.config.ClusterConfig. This method will use :meth:`kafka_cluster_type` and :meth:`kafka_cluster_name` to fetch a `ClusterConfig` using Yelp's kafka discovery mechanism. If they both aren't specified, it will fall back to creating a `ClusterConfig` from :meth:`kafka_broker_list` and :meth:`kafka_zookeeper`. The default `ClusterConfig` will point at the testing docker container. """ if ( self.kafka_cluster_type is not None and self.kafka_cluster_name is not None and not self.should_use_testing_containers ): from yelp_kafka.discovery import get_kafka_cluster # NOQA return get_kafka_cluster(self.kafka_cluster_type, 'data_pipeline-client', self.kafka_cluster_name ) else: return ClusterConfig( type='standard', name='data_pipeline', broker_list=self.kafka_broker_list, zookeeper=self.kafka_zookeeper )
def test___eq___broker_str(self): cluster_config1 = ClusterConfig( type='some_type', name='some_cluster', broker_list='kafka-cluster-1:9092,kafka-cluster-2:9092', zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,') cluster_config2 = ClusterConfig( type='some_type', name='some_cluster', broker_list='kafka-cluster-1:9092,kafka-cluster-2:9092', zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,') assert cluster_config1 == cluster_config2 # Re-order the comma separated pair of brokers and zookeeper nodes cluster_config2 = ClusterConfig( type='some_type', name='some_cluster', broker_list='kafka-cluster-2:9092,kafka-cluster-1:9092', zookeeper='zookeeper-cluster-2:2181,zookeeper-cluster-1:2181,') assert cluster_config1 == cluster_config2
def test___eq___broker_list(self): cluster_config1 = ClusterConfig( type='some_type', name='some_cluster', broker_list=['kafka-cluster-1:9092', 'kafka-cluster-2:9092'], zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,') cluster_config2 = ClusterConfig( type='some_type', name='some_cluster', broker_list=['kafka-cluster-1:9092', 'kafka-cluster-2:9092'], zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,') assert cluster_config1 == cluster_config2 # Re-ordering the list of brokers cluster_config2 = ClusterConfig( type='some_type', name='some_cluster', broker_list=['kafka-cluster-2:9092', 'kafka-cluster-1:9092'], zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,') assert cluster_config1 == cluster_config2
def test_get_cluster_by_name(self, mock_yaml): topology = TopologyConfiguration( cluster_type='mykafka', kafka_topology_path=TEST_BASE_KAFKA, ) actual_cluster = topology.get_cluster_by_name('cluster1') expected_cluster = ClusterConfig('mykafka', 'cluster1', ["mybrokerhost1:9092"], "0.1.2.3,0.2.3.4/kafka") assert expected_cluster == actual_cluster
def test___ne___broker_str(self): cluster_config1 = ClusterConfig( type='some_type', name='some_cluster', broker_list='kafka-cluster-1:9092,kafka-cluster-2:9092', zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,') # Different comma separated pair of brokers cluster_config2 = ClusterConfig( type='some_type', name='some_cluster', broker_list='kafka-cluster-2:9092,kafka-cluster-3:9092', zookeeper='zookeeper-cluster-1:2181,zookeeper-cluster-2:2181,') assert cluster_config1 != cluster_config2 # Different comma separated pair of zookeeper nodes cluster_config2 = ClusterConfig( type='some_type', name='some_cluster', broker_list='kafka-cluster-1:9092,kafka-cluster-2:9092', zookeeper='zookeeper-cluster-2:2181,zookeeper-cluster-3:2181,') assert cluster_config1 != cluster_config2
def test_get_local_cluster(self, mock_yaml): topology = TopologyConfiguration( cluster_type='mykafka', kafka_topology_path=TEST_BASE_KAFKA, ) mock_yaml.assert_called_once_with('/base/kafka_discovery/mykafka.yaml') actual_cluster = topology.get_local_cluster() expected_cluster = ClusterConfig( 'mykafka', 'cluster1', ['mybrokerhost1:9092'], '0.1.2.3,0.2.3.4/kafka', ) assert actual_cluster == expected_cluster
class TestMetadata(object): cluster_config = ClusterConfig( type='mytype', name='some_cluster', broker_list='some_list', zookeeper='some_ip' ) def test_get_topic_partition_metadata_empty(self, mock_get_metadata): mock_get_metadata.return_value = {} with mock.patch( 'kafka_utils.util.metadata.ZK', ): actual = get_topic_partition_with_error(self.cluster_config, 5) expected = set([]) assert actual == expected mock_get_metadata.asserd_called_wity('some_list') def test_get_topic_partition_metadata_no_errors(self, mock_get_metadata): mock_get_metadata.return_value = METADATA_RESPONSE_ALL_FINE with mock.patch( 'kafka_utils.util.metadata.ZK', ): actual = get_topic_partition_with_error(self.cluster_config, 5) expected = set([]) assert actual == expected def test_get_topic_partition_metadata_replica_not_available(self, mock_get_metadata): mock_get_metadata.return_value = METADATA_RESPONSE_WITH_ERRORS with mock.patch( 'kafka_utils.util.metadata.ZK', ): actual = get_topic_partition_with_error(self.cluster_config, REPLICA_NOT_AVAILABLE_ERROR) expected = {('topic0', 0), ('topic1', 1)} assert actual == expected def test_get_topic_partition_metadata_leader_not_available(self, mock_get_metadata): mock_get_metadata.return_value = METADATA_RESPONSE_WITH_ERRORS with mock.patch( 'kafka_utils.util.metadata.ZK', ): actual = get_topic_partition_with_error(self.cluster_config, LEADER_NOT_AVAILABLE_ERROR) expected = set([('topic0', 1)]) assert actual == expected
class TestZK(object): cluster_config = ClusterConfig(type='mytype', name='some_cluster', broker_list='some_list', zookeeper='some_ip') def test_create(self, mock_client): with ZK(self.cluster_config) as zk: zk.create('/kafka/consumers/some_group/offsets') zk.create('/kafka/consumers/some_group/offsets', value='some_val', acl=None, ephemeral=True, sequence=True, makepath=True) mock_obj = mock.Mock() zk.create( '/kafka/consumers/some_group/offsets', value='some_val', acl=mock_obj, ) call_list = [ mock.call('/kafka/consumers/some_group/offsets', '', None, False, False, False), mock.call('/kafka/consumers/some_group/offsets', 'some_val', None, True, True, True), mock.call('/kafka/consumers/some_group/offsets', 'some_val', mock_obj, False, False, False), ] assert mock_client.return_value.create.call_args_list == call_list def test_set(self, mock_client): with ZK(self.cluster_config) as zk: zk.set('config/topics/some_topic', 'some_val') zk.set('brokers/topics/some_topic', '{"name": "some_topic", "more": "properties"}') call_list = [ mock.call('config/topics/some_topic', 'some_val'), mock.call('brokers/topics/some_topic', '{"name": "some_topic", "more": "properties"}') ] assert mock_client.return_value.set.call_args_list == call_list def test_delete(self, mock_client): with ZK(self.cluster_config) as zk: zk.delete('/kafka/consumers/some_group/offsets', ) zk.delete('/kafka/consumers/some_group/offsets', recursive=True) call_list = [ mock.call('/kafka/consumers/some_group/offsets', recursive=False), mock.call('/kafka/consumers/some_group/offsets', recursive=True), ] assert mock_client.return_value.delete.call_args_list == call_list def test_delete_topic(self, _): with mock.patch.object(ZK, 'delete', autospec=True) as mock_delete: with ZK(self.cluster_config) as zk: zk.delete_topic( 'some_group', 'some_topic', ) mock_delete.assert_called_once_with( zk, '/consumers/some_group/offsets/some_topic', True, ) def test_get_my_subscribed_partitions(self, _): with mock.patch.object( ZK, 'get_children', autospec=True, ) as mock_children: with ZK(self.cluster_config) as zk: zk.get_my_subscribed_partitions( 'some_group', 'some_topic', ) mock_children.assert_called_once_with( zk, '/consumers/some_group/offsets/some_topic', ) def test_get_topic_config(self, mock_client): with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock(return_value=( b'{"version": 1, "config": {"cleanup.policy": "compact"}}', "Random node info that doesn't matter")) actual = zk.get_topic_config("some_topic") expected = {"version": 1, "config": {"cleanup.policy": "compact"}} assert actual == expected def test_get_topic_config_8(self, mock_client): """ Test getting configuration for topics created in Kafa prior to 0.9.0. """ with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock(side_effect=NoNodeError()) zk.get_topics = mock.Mock(return_value={"some_topic": {}}) actual = zk.get_topic_config("some_topic") expected = {"config": {}} assert actual == expected def test_get_nonexistent_topic_config(self, mock_client): """ Test getting configuration for topics that don't exist. """ with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock(side_effect=NoNodeError()) zk.get_topics = mock.Mock(return_value={}) with pytest.raises(NoNodeError): zk.get_topic_config("some_topic") def test_set_topic_config_kafka_10(self, mock_client): with mock.patch.object(ZK, 'set', autospec=True) as mock_set: with ZK(self.cluster_config) as zk: config = { "version": 1, "config": { "cleanup.policy": "compact" } } config_change = { "entity_path": "topics/some_topic", "version": 2 } zk.set_topic_config( "some_topic", config, ) serialized_config = dump_json(config) serialized_config_change = dump_json(config_change) mock_set.assert_called_once_with( zk, '/config/topics/some_topic', serialized_config, ) expected_create_call = mock.call( '/config/changes/config_change_', serialized_config_change, None, False, True, False) assert mock_client.return_value.create.call_args_list == [ expected_create_call ] def test_set_topic_config_kafka_9(self, mock_client): with mock.patch.object(ZK, 'set', autospec=True) as mock_set: with ZK(self.cluster_config) as zk: config = { "version": 1, "config": { "cleanup.policy": "compact" } } config_change = { "version": 1, "entity_type": "topics", "entity_name": "some_topic" } zk.set_topic_config("some_topic", config, (0, 9, 2)) serialized_config = dump_json(config) serialized_config_change = dump_json(config_change) mock_set.assert_called_once_with( zk, '/config/topics/some_topic', serialized_config, ) expected_create_call = mock.call( '/config/changes/config_change_', serialized_config_change, None, False, True, False) assert mock_client.return_value.create.call_args_list == [ expected_create_call ] def test_get_broker_config(self, mock_client): with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock(return_value=( b'{"version": 1, "config": {"leader.replication.throttled.rate": "42"}}', "Random node info that doesn't matter")) actual = zk.get_broker_config(0) expected = { "version": 1, "config": { "leader.replication.throttled.rate": "42" } } assert actual == expected def test_set_broker_config_kafka_10(self, mock_client): with mock.patch.object(ZK, 'set', autospec=True) as mock_set: with ZK(self.cluster_config) as zk: config = { "version": 1, "config": { "leader.replication.throttled.rate": "42" } } config_change = {"entity_path": "brokers/0", "version": 2} zk.set_broker_config(0, config) serialized_config = dump_json(config) serialized_config_change = dump_json(config_change) mock_set.assert_called_once_with( zk, '/config/brokers/0', serialized_config, ) expected_create_call = mock.call( '/config/changes/config_change_', serialized_config_change, None, False, True, False) assert mock_client.return_value.create.call_args_list == [ expected_create_call ] def test_get_topics(self, mock_client): with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock(return_value=(( b'{"version": "1", "partitions": {"0": [1, 0]}}', MockGetTopics(31000), ))) zk._fetch_partition_state = mock.Mock(return_value=(( b'{"version": "2"}', MockGetTopics(32000), ))) actual_with_fetch_state = zk.get_topics("some_topic") expected_with_fetch_state = { 'some_topic': { 'ctime': 31.0, 'partitions': { '0': { 'replicas': [1, 0], 'ctime': 32.0, 'version': '2', }, }, 'version': '1', }, } assert actual_with_fetch_state == expected_with_fetch_state zk._fetch_partition_info = mock.Mock( return_value=MockGetTopics(33000)) actual_without_fetch_state = zk.get_topics( "some_topic", fetch_partition_state=False) expected_without_fetch_state = { 'some_topic': { 'ctime': 31.0, 'partitions': { '0': { 'replicas': [1, 0], 'ctime': 33.0, }, }, 'version': '1', }, } assert actual_without_fetch_state == expected_without_fetch_state def test_get_topics_empty_cluster(self, mock_client): with ZK(self.cluster_config) as zk: zk.get_children = mock.Mock(side_effect=NoNodeError()) actual_with_no_node_error = zk.get_topics() expected_with_no_node_error = {} zk.get_children.assert_called_with("/brokers/topics") assert actual_with_no_node_error == expected_with_no_node_error def test_get_brokers_names_only(self, mock_client): with ZK(self.cluster_config) as zk: zk.get_children = mock.Mock(return_value=[1, 2, 3], ) expected = {1: None, 2: None, 3: None} actual = zk.get_brokers(names_only=True) zk.get_children.assert_called_with("/brokers/ids") assert actual == expected def test_get_brokers_with_metadata(self, mock_client): with ZK(self.cluster_config) as zk: zk.get_children = mock.Mock(return_value=[1, 2, 3], ) zk.get_broker_metadata = mock.Mock(return_value='broker', ) expected = {1: 'broker', 2: 'broker', 3: 'broker'} actual = zk.get_brokers() zk.get_children.assert_called_with("/brokers/ids") calls = zk.get_broker_metadata.mock_calls zk.get_broker_metadata.assert_has_calls(calls) assert actual == expected def test_get_brokers_empty_cluster(self, mock_client): with ZK(self.cluster_config) as zk: zk.get_children = mock.Mock(side_effect=NoNodeError()) actual_with_no_node_error = zk.get_brokers() expected_with_no_node_error = {} zk.get_children.assert_called_with("/brokers/ids") assert actual_with_no_node_error == expected_with_no_node_error def test_get_brokers_with_metadata_for_ssl(self, mock_client): with ZK(self.cluster_config) as zk: zk.get_children = mock.Mock(return_value=[1], ) zk.get = mock.Mock(return_value=( b'{"endpoints":["SSL://broker:9093"],"host":null}', None)) expected = {1: {'host': 'broker'}} actual = zk.get_brokers() assert actual[1]['host'] == expected[1]['host'] zk.get = mock.Mock(return_value=( b'{"endpoints":["INTERNAL://broker:9093","EXTERNAL://broker:9093"],"host":null}', None)) expected = {1: {'host': 'broker'}} actual = zk.get_brokers() assert actual[1]['host'] == expected[1]['host'] def test_get_brokers_with_metadata_for_sasl(self, mock_client): with ZK(self.cluster_config) as zk: zk.get_children = mock.Mock(return_value=[1], ) zk.get = mock.Mock(return_value=( b'{"endpoints":["PLAINTEXTSASL://broker:9093"],"host":null}', None)) expected = {1: {'host': 'broker'}} actual = zk.get_brokers() assert actual[1]['host'] == expected[1]['host'] def test_get_brokers_with_metadata_for_plaintext(self, mock_client): with ZK(self.cluster_config) as zk: zk.get_children = mock.Mock(return_value=[1], ) zk.get = mock.Mock( return_value=(b'{"endpoints":[],"host":"broker"}', None)) expected = {1: {'host': 'broker'}} actual = zk.get_brokers() assert actual[1]['host'] == expected[1]['host']
class TestZK(object): cluster_config = ClusterConfig(type='mytype', name='some_cluster', broker_list='some_list', zookeeper='some_ip') def test_create(self, mock_client): with ZK(self.cluster_config) as zk: zk.create('/kafka/consumers/some_group/offsets') zk.create('/kafka/consumers/some_group/offsets', value='some_val', acl=None, ephemeral=True, sequence=True, makepath=True) mock_obj = mock.Mock() zk.create( '/kafka/consumers/some_group/offsets', value='some_val', acl=mock_obj, ) call_list = [ mock.call('/kafka/consumers/some_group/offsets', '', None, False, False, False), mock.call('/kafka/consumers/some_group/offsets', 'some_val', None, True, True, True), mock.call('/kafka/consumers/some_group/offsets', 'some_val', mock_obj, False, False, False), ] assert mock_client.return_value.create.call_args_list == call_list def test_set(self, mock_client): with ZK(self.cluster_config) as zk: zk.set('config/topics/some_topic', 'some_val') zk.set('brokers/topics/some_topic', '{"name": "some_topic", "more": "properties"}') call_list = [ mock.call('config/topics/some_topic', 'some_val'), mock.call('brokers/topics/some_topic', '{"name": "some_topic", "more": "properties"}') ] assert mock_client.return_value.set.call_args_list == call_list def test_delete(self, mock_client): with ZK(self.cluster_config) as zk: zk.delete('/kafka/consumers/some_group/offsets', ) zk.delete('/kafka/consumers/some_group/offsets', recursive=True) call_list = [ mock.call('/kafka/consumers/some_group/offsets', recursive=False), mock.call('/kafka/consumers/some_group/offsets', recursive=True), ] assert mock_client.return_value.delete.call_args_list == call_list def test_delete_topic_partitions(self, mock_client): with mock.patch.object(ZK, 'delete', autospec=True) as mock_delete: with ZK(self.cluster_config) as zk: zk.delete_topic_partitions('some_group', 'some_topic', [0, 1, 2]) call_list = [ mock.call(zk, '/consumers/some_group/offsets/some_topic/0'), mock.call(zk, '/consumers/some_group/offsets/some_topic/1'), mock.call(zk, '/consumers/some_group/offsets/some_topic/2'), ] assert mock_delete.call_args_list == call_list def test_delete_topic(self, _): with mock.patch.object(ZK, 'delete', autospec=True) as mock_delete: with ZK(self.cluster_config) as zk: zk.delete_topic( 'some_group', 'some_topic', ) mock_delete.assert_called_once_with( zk, '/consumers/some_group/offsets/some_topic', True, ) def test_get_my_subscribed_partitions(self, _): with mock.patch.object( ZK, 'get_children', autospec=True, ) as mock_children: with ZK(self.cluster_config) as zk: zk.get_my_subscribed_partitions( 'some_group', 'some_topic', ) mock_children.assert_called_once_with( zk, '/consumers/some_group/offsets/some_topic', ) def test_get_topic_config(self, mock_client): with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock(return_value=( b'{"version": 1, "config": {"cleanup.policy": "compact"}}', "Random node info that doesn't matter")) actual = zk.get_topic_config("some_topic") expected = {"version": 1, "config": {"cleanup.policy": "compact"}} assert actual == expected def test_set_topic_config_kafka_10(self, mock_client): with mock.patch.object(ZK, 'set', autospec=True) as mock_set: with ZK(self.cluster_config) as zk: config = { "version": 1, "config": { "cleanup.policy": "compact" } } config_change = { "entity_path": "topics/some_topic", "version": 2 } zk.set_topic_config( "some_topic", config, ) serialized_config = dump_json(config) serialized_config_change = dump_json(config_change) mock_set.assert_called_once_with( zk, '/config/topics/some_topic', serialized_config, ) expected_create_call = mock.call( '/config/changes/config_change_', serialized_config_change, None, False, True, False) assert mock_client.return_value.create.call_args_list == [ expected_create_call ] def test_set_topic_config_kafka_9(self, mock_client): with mock.patch.object(ZK, 'set', autospec=True) as mock_set: with ZK(self.cluster_config) as zk: config = { "version": 1, "config": { "cleanup.policy": "compact" } } config_change = { "version": 1, "entity_type": "topics", "entity_name": "some_topic" } zk.set_topic_config("some_topic", config, (0, 9, 2)) serialized_config = dump_json(config) serialized_config_change = dump_json(config_change) mock_set.assert_called_once_with( zk, '/config/topics/some_topic', serialized_config, ) expected_create_call = mock.call( '/config/changes/config_change_', serialized_config_change, None, False, True, False) assert mock_client.return_value.create.call_args_list == [ expected_create_call ]
main, 'read_cluster_status', side_effect=read_cluster_state_values([], (100, 0)), autospec=True, ) @mock.patch('time.sleep', autospec=True) def test_wait_for_stable_cluster_timeout(mock_sleep, mock_read): with pytest.raises(main.WaitTimeoutException): main.wait_for_stable_cluster([], 1, "", None, None, 5, 3, 100) assert mock_read.call_count == 21 assert mock_sleep.mock_calls == [mock.call(5)] * 20 cluster_config = ClusterConfig(type='mytype', name='some_cluster', broker_list='some_list', zookeeper='some_ip') @mock.patch('kafka_utils.util.zookeeper.KazooClient', autospec=True) @mock.patch.object(ZK, 'get_brokers', side_effect=[{ 2: { 'host': 'broker2' }, 3: { 'host': 'broker3' }, 1: { 'host': 'broker1'
class TestZK(object): cluster_config = ClusterConfig( type='mytype', name='some_cluster', broker_list='some_list', zookeeper='some_ip' ) def test_create(self, mock_client): with ZK(self.cluster_config) as zk: zk.create( '/kafka/consumers/some_group/offsets' ) zk.create( '/kafka/consumers/some_group/offsets', value='some_val', acl=None, ephemeral=True, sequence=True, makepath=True ) mock_obj = mock.Mock() zk.create( '/kafka/consumers/some_group/offsets', value='some_val', acl=mock_obj, ) call_list = [ mock.call( '/kafka/consumers/some_group/offsets', '', None, False, False, False ), mock.call( '/kafka/consumers/some_group/offsets', 'some_val', None, True, True, True ), mock.call( '/kafka/consumers/some_group/offsets', 'some_val', mock_obj, False, False, False ), ] assert mock_client.return_value.create.call_args_list == call_list def test_set(self, mock_client): with ZK(self.cluster_config) as zk: zk.set( 'config/topics/some_topic', 'some_val' ) zk.set( 'brokers/topics/some_topic', '{"name": "some_topic", "more": "properties"}' ) call_list = [ mock.call( 'config/topics/some_topic', 'some_val' ), mock.call( 'brokers/topics/some_topic', '{"name": "some_topic", "more": "properties"}' ) ] assert mock_client.return_value.set.call_args_list == call_list def test_delete(self, mock_client): with ZK(self.cluster_config) as zk: zk.delete( '/kafka/consumers/some_group/offsets', ) zk.delete( '/kafka/consumers/some_group/offsets', recursive=True ) call_list = [ mock.call( '/kafka/consumers/some_group/offsets', recursive=False ), mock.call( '/kafka/consumers/some_group/offsets', recursive=True ), ] assert mock_client.return_value.delete.call_args_list == call_list def test_delete_topic_partitions(self, mock_client): with mock.patch.object( ZK, 'delete', autospec=True ) as mock_delete: with ZK(self.cluster_config) as zk: zk.delete_topic_partitions( 'some_group', 'some_topic', [0, 1, 2] ) call_list = [ mock.call( zk, '/consumers/some_group/offsets/some_topic/0' ), mock.call( zk, '/consumers/some_group/offsets/some_topic/1' ), mock.call( zk, '/consumers/some_group/offsets/some_topic/2' ), ] assert mock_delete.call_args_list == call_list def test_delete_topic(self, _): with mock.patch.object( ZK, 'delete', autospec=True ) as mock_delete: with ZK(self.cluster_config) as zk: zk.delete_topic( 'some_group', 'some_topic', ) mock_delete.assert_called_once_with( zk, '/consumers/some_group/offsets/some_topic', True, ) def test_get_my_subscribed_partitions(self, _): with mock.patch.object( ZK, 'get_children', autospec=True, ) as mock_children: with ZK(self.cluster_config) as zk: zk.get_my_subscribed_partitions( 'some_group', 'some_topic', ) mock_children.assert_called_once_with( zk, '/consumers/some_group/offsets/some_topic', ) def test_get_topic_config(self, mock_client): with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock( return_value=( b'{"version": 1, "config": {"cleanup.policy": "compact"}}', "Random node info that doesn't matter" ) ) actual = zk.get_topic_config("some_topic") expected = {"version": 1, "config": {"cleanup.policy": "compact"}} assert actual == expected def test_get_topic_config_8(self, mock_client): """ Test getting configuration for topics created in Kafa prior to 0.9.0. """ with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock(side_effect=NoNodeError()) zk.get_topics = mock.Mock(return_value={"some_topic": {}}) actual = zk.get_topic_config("some_topic") expected = {"config": {}} assert actual == expected def test_get_nonexistent_topic_config(self, mock_client): """ Test getting configuration for topics that don't exist. """ with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock(side_effect=NoNodeError()) zk.get_topics = mock.Mock(return_value={}) with pytest.raises(NoNodeError): zk.get_topic_config("some_topic") def test_set_topic_config_kafka_10(self, mock_client): with mock.patch.object( ZK, 'set', autospec=True ) as mock_set: with ZK(self.cluster_config) as zk: config = {"version": 1, "config": {"cleanup.policy": "compact"}} config_change = {"entity_path": "topics/some_topic", "version": 2} zk.set_topic_config( "some_topic", config, ) serialized_config = dump_json(config) serialized_config_change = dump_json(config_change) mock_set.assert_called_once_with( zk, '/config/topics/some_topic', serialized_config, ) expected_create_call = mock.call( '/config/changes/config_change_', serialized_config_change, None, False, True, False ) assert mock_client.return_value.create.call_args_list == [expected_create_call] def test_set_topic_config_kafka_9(self, mock_client): with mock.patch.object( ZK, 'set', autospec=True ) as mock_set: with ZK(self.cluster_config) as zk: config = {"version": 1, "config": {"cleanup.policy": "compact"}} config_change = {"version": 1, "entity_type": "topics", "entity_name": "some_topic"} zk.set_topic_config( "some_topic", config, (0, 9, 2) ) serialized_config = dump_json(config) serialized_config_change = dump_json(config_change) mock_set.assert_called_once_with( zk, '/config/topics/some_topic', serialized_config, ) expected_create_call = mock.call( '/config/changes/config_change_', serialized_config_change, None, False, True, False ) assert mock_client.return_value.create.call_args_list == [expected_create_call] def test_get_topics(self, mock_client): with ZK(self.cluster_config) as zk: zk.zk.get = mock.Mock( return_value=( ( b'{"version": "1", "partitions": {"0": [1, 0]}}', TestGetTopics(31000), ) ) ) zk._fetch_partition_state = mock.Mock( return_value=( ( b'{"version": "2"}', TestGetTopics(32000), ) ) ) actual_with_fetch_state = zk.get_topics("some_topic") expected_with_fetch_state = { 'some_topic': { 'ctime': 31.0, 'partitions': { '0': { 'replicas': [1, 0], 'ctime': 32.0, 'version': '2', }, }, 'version': '1', }, } assert actual_with_fetch_state == expected_with_fetch_state zk._fetch_partition_info = mock.Mock( return_value=TestGetTopics(33000) ) actual_without_fetch_state = zk.get_topics("some_topic", fetch_partition_state=False) expected_without_fetch_state = { 'some_topic': { 'ctime': 31.0, 'partitions': { '0': { 'replicas': [1, 0], 'ctime': 33.0, }, }, 'version': '1', }, } assert actual_without_fetch_state == expected_without_fetch_state