def test_get_kafka_cluster_invalid(
mock_kafka_discovery_client,
mock_response_obj,
mock_http_err,
):
mock_kafka_discovery_client.return_value.v1.getClustersWithName.\
return_value.result.side_effect = mock_http_err
with pytest.raises(InvalidClusterTypeOrNameError):
discovery.get_kafka_cluster('invalid-type', 'client-1', 'cluster1')
def cluster_config(self):
"""Returns a yelp_kafka.config.ClusterConfig.
This method will use :meth:`kafka_cluster_type` and
:meth:`kafka_cluster_name` to fetch a `ClusterConfig` using Yelp's
kafka discovery mechanism. If they both aren't specified, it will
fall back to creating a `ClusterConfig` from :meth:`kafka_broker_list`
and :meth:`kafka_zookeeper`. The default `ClusterConfig` will point
at the testing docker container.
"""
if (
self.kafka_cluster_type is not None and
self.kafka_cluster_name is not None and
not self.should_use_testing_containers
):
from yelp_kafka.discovery import get_kafka_cluster # NOQA
return get_kafka_cluster(self.kafka_cluster_type,
'data_pipeline-client',
self.kafka_cluster_name
)
else:
return ClusterConfig(
type='standard',
name='data_pipeline',
broker_list=self.kafka_broker_list,
zookeeper=self.kafka_zookeeper
)
def cluster_config(self):
"""Returns a yelp_kafka.config.ClusterConfig.
This method will use :meth:`kafka_cluster_type` and
:meth:`kafka_cluster_name` to fetch a `ClusterConfig` using Yelp's
kafka discovery mechanism. If they both aren't specified, it will
fall back to creating a `ClusterConfig` from :meth:`kafka_broker_list`
and :meth:`kafka_zookeeper`. The default `ClusterConfig` will point
at the testing docker container.
"""
if (
self.kafka_cluster_type is not None and
self.kafka_cluster_name is not None and
not self.should_use_testing_containers
):
from yelp_kafka.discovery import get_kafka_cluster # NOQA
return get_kafka_cluster(self.kafka_cluster_type,
'data_pipeline-client',
self.kafka_cluster_name
)
else:
return ClusterConfig(
type='standard',
name='data_pipeline',
broker_list=self.kafka_broker_list,
zookeeper=self.kafka_zookeeper
)
def _region_cluster_config(self):
""" The ClusterConfig for Kafka cluster to connect to. If cluster_name
is not specified, it will default to the value set in Config"""
if self.cluster_name:
return discovery.get_kafka_cluster(cluster_type=self.cluster_type,
client_id=self.client_name,
cluster_name=self.cluster_name)
else:
return get_config().cluster_config
def test_get_kafka_cluster(mock_kafka_discovery_client, mock_response_obj, mock_clusters):
mock_kafka_discovery_client.return_value.v1.getClustersWithName.\
return_value.result.return_value = mock_response_obj
cluster_config = discovery.get_kafka_cluster('type1', 'client-1', 'cluster1')
assert cluster_config == mock_clusters[0]
mock_kafka_discovery_client.return_value.v1.getClustersWithName\
.assert_called_with(
type='type1',
kafka_cluster_name='cluster1',
)
def _region_cluster_config(self):
""" The ClusterConfig for Kafka cluster to connect to. If cluster_name
is not specified, it will default to the value set in Config"""
if self.cluster_name:
return discovery.get_kafka_cluster(
cluster_type=self.cluster_type,
client_id=self.client_name,
cluster_name=self.cluster_name
)
else:
return get_config().cluster_config