def get_all_clusters(cluster_type, client_id):
"""Get a list of (cluster_name, cluster_config)
for the available kafka clusters in the ecosystem at Yelp.
:param cluster_type: kafka cluster type
(ex.'scribe' or 'standard').
:type cluster_type: string
:param client_id: name of the client making the request. Usually
the same client id used to create the Kafka connection.
:type client_id: string
:returns: list of py:class:`yelp_kafka.config.ClusterConfig`
"""
client = get_kafka_discovery_client(client_id)
try:
cluster_names = client.v1.getClustersAll(cluster_type).result()
except HTTPError as e:
log.exception(
"Failure while fetching clusters for cluster type:{clustertype}"
.format(clustertype=cluster_type),
)
raise InvalidClusterType(e.response.text)
return [
get_kafka_cluster(cluster_type, client_id, cluster_name)
for cluster_name in cluster_names
]
def get_superregion_logs_regex(client_id, regex, superregion=None):
"""Get the kafka cluster logs for given superregion and topic-regex at Yelp.
If no superregion is given, we default to local superregion.
:param client_id: name of the client making the logs request. Usually
the same client id used to create the Kafka connection.
:type client_id: string
:param superregion: superregion name defaulting to caller's local superregion.
:type superregion: string
:param regex: Log/stream name regex
:type regex: string
:returns: [([topics], cluster)]
:raises InvalidLogOrSuperegionError: :py:class:`yelp_kafka.error.InvalidLogOrRegionError`
upon failure fetching logs from superregion.
"""
if not superregion:
superregion = _get_local_superregion()
client = get_kafka_discovery_client(client_id)
try:
result = client.v1.getLogsForSuperregionWithRegex(
superregion=superregion,
regex=regex,
).result()
return parse_as_logs_topics(result)
except HTTPError as e:
log.exception(
"Failure while fetching logs for superregion:{superregion}"
.format(superregion=superregion),
)
raise InvalidLogOrSuperregionError(e.response.text)
def get_kafka_cluster(cluster_type, client_id, cluster_name):
"""Get a :py:class:`yelp_kafka.config.ClusterConfig` for a given
cluster-type and name at Yelp.
:param cluster_type: kafka cluster type (ex.'scribe' or 'standard').
:type cluster_type: string
:param client_id: name of the client making the discovery request. Usually
the same client id used to create the Kafka connection.
:type client_id: string
:param cluster_name: name of the cluster (ex.'uswest1-devc').
:type cluster_name: string
:returns: :py:class:`yelp_kafka.config.ClusterConfig`
"""
client = get_kafka_discovery_client(client_id)
try:
result = client.v1.getClustersWithName(
type=cluster_type,
kafka_cluster_name=cluster_name,
).result()
return parse_as_cluster_config(result)
except HTTPError as e:
log.exception(
"Failure while fetching kafka cluster for cluster-type:{type}, cluster-name"
":{cluster_name}.".format(type=cluster_type, cluster_name=cluster_name),
)
raise InvalidClusterTypeOrNameError(e.response.text)
def get_superregion_cluster(cluster_type, client_id, superregion=None):
"""Get the kafka cluster for given superregion at Yelp. If no region is specified,
we default to local superregion.
:param cluster_type: kafka cluster type (ex.'scribe' or 'standard').
:type cluster_type: string
:param client_id: name of the client making the discovery request. Usually
the same client id used to create the Kafka connection.
:type client_id: string
:param superregion: region name for which kafka cluster is desired.
:type superregion: string
:returns: py:class:`yelp_kafka.config.ClusterConfig`
"""
if not superregion:
superregion = _get_local_superregion()
client = get_kafka_discovery_client(client_id)
try:
result = client.v1.getClustersWithSuperregion(
type=cluster_type,
superregion=superregion,
).result()
return parse_as_cluster_config(result)
except HTTPError as e:
log.exception(
"Failure while fetching kafka cluster for cluster-type:{type}, "
"superregion :{superregion}.".format(type=cluster_type, superregion=superregion),
)
raise InvalidClusterTypeOrSuperregionError(e.response.text)
def get_all_logs_regions(client_id):
"""Get a list of all regions that have logs at Yelp.
:param client_id: name of the client making the request. Usually
the same client id used to create the Kafka connection.
:type client_id: string
:returns: list of strings
"""
client = get_kafka_discovery_client(client_id)
try:
regions = client.v1.getLogsRegions().result()
except HTTPError as e:
log.exception("Failure while fetching list of all logs' regions")
raise DiscoveryError(e.response.text)
return regions
def test_get_kafka_discovery_client(mock_swagger_yaml):
if getattr(yelp_kafka.config, 'SmartStackClient', None) is None:
return
with mock.patch(
"yelp_kafka.config.SmartStackClient",
autospec=True,
) as mock_client:
with mock.patch(
"yelp_kafka.config.SwaggerClient",
autospec=True,
) as mock_swagger:
with mock.patch(
'yelp_kafka.config.RequestsClient',
autospec=True,
) as mock_request:
with mock.patch(
'yelp_kafka.config.ZipkinClientDecorator',
autospec=True,
) as mock_zipkin_wrapper:
mock_swagger.from_url.return_value = mock.sentinel.swagger_client
mock_zipkin_wrapper.return_value = mock.sentinel.zipkin_client
mock_client.return_value = mock.sentinel.client
mock_request.return_value = mock.sentinel.request
actual_client = get_kafka_discovery_client('myclientid')
assert actual_client == mock.sentinel.client
actual_args, _ = mock_zipkin_wrapper.call_args
assert actual_args[0] == mock.sentinel.swagger_client
mock_swagger.from_url.assert_called_once_with(
u'http://host2:2222/swagger.json', # See conftest.py
mock.sentinel.request,
)
assert mock_client.call_count == 1
actual_args, actual_kwargs = mock_client.call_args
assert actual_args[0] == mock.sentinel.zipkin_client
assert actual_kwargs == {
'client_name': 'myclientid',
'service_name': 'kafka_discovery',
}