def _detect_consumer_via_kafka(self):
"""Detect consumers groups using kafka-consumer-groups"""
log.info("\tDetecting kafka consumers with {:s} command".format(
_KAFKA_CONSUMER_GROUP_COMMAND))
try:
output = check_output([
_KAFKA_CONSUMER_GROUP_COMMAND,
'--zookeeper',
self.zk_url,
'--list'
], stderr=STDOUT)
consumers = {}
consumer_groups = output.splitlines()
for consumer_group in consumer_groups:
output = check_output([
_KAFKA_CONSUMER_GROUP_COMMAND,
'--zookeeper',
self.zk_url,
'--describe',
'--group',
consumer_group
], stderr=STDOUT)
lines = output.splitlines()
topics = {}
for it, line in enumerate(reversed(lines)):
if it == len(lines) - 1 or not line:
break
values = line.split(',')
# There will be always 7 values in output
# after splitting the line by ,
if (values and len(values)
== _CONSUMER_GROUP_COMMAND_LINE_VALUES_LEN):
topics[values[_VIA_KAFKA_TOPIC_INDEX].strip()] = []
if len(topics.keys()):
consumers[consumer_group] = topics
return consumers
except Exception as ex:
log.warn(('Failed to retrieve consumers '
'with kafka-consumer-groups.sh. Error is %s'), ex)
return None
def _detect_consumer_via_kafka(self):
"""Detect consumers groups using kafka-consumer-groups"""
log.info("\tDetecting kafka consumers with {:s} command".format(
self._kafka_consumer_bin))
try:
output = check_output([
self._kafka_consumer_bin,
'--zookeeper',
self.zk_url,
'--list'
], stderr=STDOUT)
consumers = {}
consumer_groups = output.splitlines()
for consumer_group in consumer_groups:
output = check_output([
self._kafka_consumer_bin,
'--zookeeper',
self.zk_url,
'--describe',
'--group',
consumer_group
], stderr=STDOUT)
lines = output.splitlines()
topics = {}
for it, line in enumerate(reversed(lines)):
if it == len(lines) - 1 or not line:
break
values = line.split(',')
# There will be always 7 values in output
# after splitting the line by ,
if (values and len(values)
== _CONSUMER_GROUP_COMMAND_LINE_VALUES_LEN):
topics[values[_VIA_KAFKA_TOPIC_INDEX].strip()] = []
if len(topics.keys()):
consumers[consumer_group] = topics
return consumers
except Exception as ex:
log.warn(('Failed to retrieve consumers '
'with kafka-consumer-groups.sh. Error is %s'), ex)
return None
def _ls_zookeeper(self, path):
""" Do a ls on the given zookeeper path.
I am using the local command line kafka rather than kazoo because it doesn't make sense to
have kazoo as a dependency only for detection.
"""
zk_shell = ['/opt/kafka/bin/zookeeper-shell.sh', self.zk_url, 'ls', path]
try:
output = check_output(zk_shell)
except CalledProcessError:
log.error('Error running the zookeeper shell to list path %s' % path)
raise
# The last line is like '[item1, item2, item3]'
return [entry.strip() for entry in output.splitlines()[-1].strip('[]').split(',')]
def _ls_zookeeper(self, path):
"""Do a ls on the given zookeeper path.
I am using the local command line kafka rather than kazoo because it doesn't make sense to
have kazoo as a dependency only for detection.
"""
zk_shell = ['/opt/kafka/bin/zookeeper-shell.sh', self.zk_url, 'ls', path]
try:
output = check_output(zk_shell, stderr=STDOUT)
except CalledProcessError:
log.error('Error running the zookeeper shell to list path %s' % path)
raise
# The last line is like '[item1, item2, item3]', '[]' or an error message not starting with [
last_line = output.splitlines()[-1]
if len(last_line) == 2 or last_line[0] != '[':
return []
else:
return [entry.strip() for entry in last_line.strip('[]').split(',')]
def _ls_zookeeper(self, path):
""" Do a ls on the given zookeeper path.
I am using the local command line kafka rather than kazoo because it doesn't make sense to
have kazoo as a dependency only for detection.
"""
zk_shell = [
'/opt/kafka/bin/zookeeper-shell.sh', self.zk_url, 'ls', path
]
try:
output = check_output(zk_shell)
except CalledProcessError:
log.error('Error running the zookeeper shell to list path %s' %
path)
raise
# The last line is like '[item1, item2, item3]'
return [
entry.strip()
for entry in output.splitlines()[-1].strip('[]').split(',')
]
def _ls_zookeeper(self, path):
"""Do a ls on the given zookeeper path.
I am using the local command line kafka rather than kazoo because it doesn't make
sense to have kazoo as a dependency only for detection.
"""
zk_shell = [self._zookeeper_consumer_bin, self.zk_url, 'ls', path]
try:
output = check_output(zk_shell, stderr=STDOUT)
except CalledProcessError:
log.error('Error running the zookeeper shell to list path %s',
path)
raise
# The last line is like '[item1, item2, item3]', '[]' or an error message
# not starting with [
last_line = output.splitlines()[-1]
if len(last_line) == 2 or last_line[0] != '[':
return []
else:
return [entry.strip() for entry in last_line.strip('[]').split(',')]