Example #1
0
    def _detect_consumer_via_kafka(self):
        """Detect consumers groups using kafka-consumer-groups"""
        log.info("\tDetecting kafka consumers with {:s} command".format(
            _KAFKA_CONSUMER_GROUP_COMMAND))
        try:
            output = check_output([
                _KAFKA_CONSUMER_GROUP_COMMAND,
                '--zookeeper',
                self.zk_url,
                '--list'
            ], stderr=STDOUT)

            consumers = {}
            consumer_groups = output.splitlines()

            for consumer_group in consumer_groups:
                output = check_output([
                    _KAFKA_CONSUMER_GROUP_COMMAND,
                    '--zookeeper',
                    self.zk_url,
                    '--describe',
                    '--group',
                    consumer_group
                ], stderr=STDOUT)

                lines = output.splitlines()
                topics = {}
                for it, line in enumerate(reversed(lines)):
                    if it == len(lines) - 1 or not line:
                        break
                    values = line.split(',')
                    # There will be always 7 values in output
                    # after splitting the line by ,
                    if (values and len(values)
                            == _CONSUMER_GROUP_COMMAND_LINE_VALUES_LEN):
                        topics[values[_VIA_KAFKA_TOPIC_INDEX].strip()] = []
                if len(topics.keys()):
                    consumers[consumer_group] = topics

            return consumers
        except Exception as ex:
            log.warn(('Failed to retrieve consumers '
                      'with kafka-consumer-groups.sh. Error is %s'), ex)
        return None
Example #2
0
    def _detect_consumer_via_kafka(self):
        """Detect consumers groups using kafka-consumer-groups"""
        log.info("\tDetecting kafka consumers with {:s} command".format(
            self._kafka_consumer_bin))
        try:
            output = check_output([
                self._kafka_consumer_bin,
                '--zookeeper',
                self.zk_url,
                '--list'
            ], stderr=STDOUT)

            consumers = {}
            consumer_groups = output.splitlines()

            for consumer_group in consumer_groups:
                output = check_output([
                    self._kafka_consumer_bin,
                    '--zookeeper',
                    self.zk_url,
                    '--describe',
                    '--group',
                    consumer_group
                ], stderr=STDOUT)

                lines = output.splitlines()
                topics = {}
                for it, line in enumerate(reversed(lines)):
                    if it == len(lines) - 1 or not line:
                        break
                    values = line.split(',')
                    # There will be always 7 values in output
                    # after splitting the line by ,
                    if (values and len(values)
                            == _CONSUMER_GROUP_COMMAND_LINE_VALUES_LEN):
                        topics[values[_VIA_KAFKA_TOPIC_INDEX].strip()] = []
                if len(topics.keys()):
                    consumers[consumer_group] = topics

            return consumers
        except Exception as ex:
            log.warn(('Failed to retrieve consumers '
                      'with kafka-consumer-groups.sh. Error is %s'), ex)
        return None
Example #3
0
    def _ls_zookeeper(self, path):
        """ Do a ls on the given zookeeper path.
            I am using the local command line kafka rather than kazoo because it doesn't make sense to
            have kazoo as a dependency only for detection.
        """
        zk_shell = ['/opt/kafka/bin/zookeeper-shell.sh', self.zk_url, 'ls', path]
        try:
            output = check_output(zk_shell)
        except CalledProcessError:
            log.error('Error running the zookeeper shell to list path %s' % path)
            raise

        # The last line is like '[item1, item2, item3]'
        return [entry.strip() for entry in output.splitlines()[-1].strip('[]').split(',')]
    def _ls_zookeeper(self, path):
        """Do a ls on the given zookeeper path.
           I am using the local command line kafka rather than kazoo because it doesn't make sense to
           have kazoo as a dependency only for detection.
        """
        zk_shell = ['/opt/kafka/bin/zookeeper-shell.sh', self.zk_url, 'ls', path]
        try:
            output = check_output(zk_shell, stderr=STDOUT)
        except CalledProcessError:
            log.error('Error running the zookeeper shell to list path %s' % path)
            raise

        # The last line is like '[item1, item2, item3]', '[]' or an error message not starting with [
        last_line = output.splitlines()[-1]
        if len(last_line) == 2 or last_line[0] != '[':
            return []
        else:
            return [entry.strip() for entry in last_line.strip('[]').split(',')]
Example #5
0
    def _ls_zookeeper(self, path):
        """ Do a ls on the given zookeeper path.
            I am using the local command line kafka rather than kazoo because it doesn't make sense to
            have kazoo as a dependency only for detection.
        """
        zk_shell = [
            '/opt/kafka/bin/zookeeper-shell.sh', self.zk_url, 'ls', path
        ]
        try:
            output = check_output(zk_shell)
        except CalledProcessError:
            log.error('Error running the zookeeper shell to list path %s' %
                      path)
            raise

        # The last line is like '[item1, item2, item3]'
        return [
            entry.strip()
            for entry in output.splitlines()[-1].strip('[]').split(',')
        ]
Example #6
0
    def _ls_zookeeper(self, path):
        """Do a ls on the given zookeeper path.
           I am using the local command line kafka rather than kazoo because it doesn't make
           sense to have kazoo as a dependency only for detection.
        """
        zk_shell = [self._zookeeper_consumer_bin, self.zk_url, 'ls', path]
        try:
            output = check_output(zk_shell, stderr=STDOUT)
        except CalledProcessError:
            log.error('Error running the zookeeper shell to list path %s',
                      path)
            raise

        # The last line is like '[item1, item2, item3]', '[]' or an error message
        # not starting with [
        last_line = output.splitlines()[-1]
        if len(last_line) == 2 or last_line[0] != '[':
            return []
        else:
            return [entry.strip() for entry in last_line.strip('[]').split(',')]