Esempio n. 1
0
    def get(self, search_string=None):
        """
        Get a config value using a search string
        :param search_string: Config param separated with a dot '.'
        :type search_string: str
        :return: The value of the config element if it exists, raise an exception otherwise
        """
        if search_string is None:
            return self._config_content

        if isinstance(search_string, str):
            param_array = search_string.split(".")
            performed_param_str = None
            ret = self._config_content

            for param in param_array:
                performed_param_str = performed_param_str + "." + param if performed_param_str is not None else param
                if param in ret:
                    ret = ret[param]
                else:
                    raise KafkaIotException("Can't find param %s in config file" % performed_param_str)

            return ret

        else:
            raise KafkaIotException("Given parameter for search_string must be a string! %s given instead" %
                                    str(type(search_string)))
Esempio n. 2
0
    def delete_topic(self, topic_name, check_topic_existence=True):
        """
        Attempt to delete a given topic.
        This will have no effect if delete.topic.enable is not set to true in kafka config
        :param topic_name: The name of the topic to delete
        :type topic_name: str
        :param check_topic_existence: Check if the topic exists before trying to delete it
        :type check_topic_existence: bool
        """
        if check_topic_existence:
            if topic_name not in self.list_topic_name():
                raise KafkaIotException("Topic \"%s\" doesn't exist" %
                                        topic_name)

        p = subprocess.Popen([
            self._get_topic_script(), "--delete", "--zookeeper",
            self._zookeeper_full_address, "--topic", topic_name
        ],
                             stdout=subprocess.PIPE)

        while True:
            line = p.stdout.readline()
            if line != b'':
                line = line.decode(KafkaDriver.DEFAULT_ENCODING).replace(
                    "\n", "")
                log_console_output("DELETE TOPIC", line)
            else:
                break
Esempio n. 3
0
def check_kafka_right_in_topic_name_list(kafka_rights, topic_name_list):
    """
    Check kafka right topics exists in topic list
    :param kafka_rights: Dictionary with read and write topic list
    :type kafka_rights: dict
    :param topic_name_list: List of topic name
    :type topic_name_list: list[str]
    :return: True if all the topics in kafka rights are in topic name list. Raise an exception otherwise
    """
    for read_topic in kafka_rights["READ"]:
        if read_topic != "*" and read_topic not in topic_name_list:
            raise KafkaIotException(
                "READ topic \"%s\" is not declared in topic list" % read_topic)

    for write_topic in kafka_rights["WRITE"]:
        if write_topic != "*" and write_topic not in topic_name_list:
            raise KafkaIotException(
                "WRITE topic \"%s\" is not declared in topic list" %
                write_topic)
Esempio n. 4
0
    def alter_topic(self,
                    topic_name,
                    partition_number=-1,
                    check_topic_existence=True):
        """
        Alter a topic replication factor and / or partition number
        :param topic_name: Name of the topic to alter
        :type topic_name: str
        :param partition_number: New partition number to be applied. No effect when set to -1
        :type partition_number: int
        :param check_topic_existence: Check if the topic exists before trying to delete it
        :type check_topic_existence: bool
        """
        if check_topic_existence:
            if topic_name not in self.list_topic_name():
                raise KafkaIotException("Topic \"%s\" doesn't exist" %
                                        topic_name)

        topic = self.describe_topic(topic_name, False)

        if topic.partition_number != partition_number > 0:
            if partition_number < topic.partition_number:  # If the number of partition has te be decreased
                self.delete_topic(topic_name, False)
                self.create_topic(topic_name, topic.replication_factor,
                                  partition_number)
                logging.warning(
                    "Topic \"%s\" has been deleted and re-created in order to reduce its partition number "
                    "from %d to %d" %
                    (topic_name, topic.partition_number, partition_number))
            else:  # Increase the number of partition
                p = subprocess.Popen([
                    self._get_topic_script(), "--alter", "--zookeeper",
                    self._zookeeper_full_address, "--topic", topic_name,
                    "--partitions",
                    str(partition_number)
                ],
                                     stdout=subprocess.PIPE)

                while True:
                    line = p.stdout.readline()
                    if line != b'':
                        line = line.decode(
                            KafkaDriver.DEFAULT_ENCODING).replace("\n", "")
                        log_console_output("ALTER TOPIC PARTITION", line)
                    else:
                        break
Esempio n. 5
0
def init_logger(log_level, log_location, app_name):
    # Getting log level
    log_level = _define_log_level(log_level)

    # Modify logger log level
    logger = logging.getLogger()
    logger.setLevel(log_level)

    # Set file formatter
    formatter = logging.Formatter("%(asctime)s :: %(levelname)s :: " +
                                  app_name + " ::  %(message)s")
    log_filename = "%s_log.txt" % app_name
    log_location = expand_var_and_user(log_location)

    if os.path.isdir(log_location):
        log_file_path = os.path.join(log_location, log_filename)
        need_roll = os.path.isfile(log_file_path)

        # Redirect logs into a log file
        file_handler = RotatingFileHandler(log_file_path,
                                           backupCount=10,
                                           maxBytes=2 * 1024 * 1024)
        file_handler.setLevel(log_level)
        file_handler.setFormatter(formatter)
        if need_roll:
            file_handler.doRollover()
        logger.addHandler(file_handler)

        # Redirect logs into the user console
        stream_handler = logging.StreamHandler()
        stream_handler.setLevel(log_level)
        stream_handler.setFormatter(formatter)
        logger.addHandler(stream_handler)
    else:
        raise KafkaIotException(
            "Log directory: %s does not exist, create it before relaunching the program"
            % log_location)
Esempio n. 6
0
    def create_topic(self,
                     topic_name,
                     replication_factor,
                     partitions,
                     check_topic_existence=True):
        """
        Attempt to create a topic
        :param topic_name: The name of the topic to be created
        :type topic_name: str
        :param replication_factor: The replication factor of the topic
        :type replication_factor: int
        :param partitions: The number of partition of the topic
        :type partitions: int
        :param check_topic_existence: Check if the topic exists before trying to create it
        """
        if check_topic_existence:
            if topic_name in self.list_topic_name():
                raise KafkaIotException("Topic \"%s\" already exists" %
                                        topic_name)

        p = subprocess.Popen([
            self._get_topic_script(), "--create", "--zookeeper",
            self._zookeeper_full_address, "--replication-factor",
            str(replication_factor), "--partitions",
            str(partitions), "--topic", topic_name
        ],
                             stdout=subprocess.PIPE)

        while True:
            line = p.stdout.readline()
            if line != b'':
                line = line.decode(KafkaDriver.DEFAULT_ENCODING).replace(
                    "\n", "")
                log_console_output("CREATE TOPIC", line)
            else:
                break
Esempio n. 7
0
    for app in sys_conf.get("APP_LIST"):
        check_fields_in_dict(app, ["NAME", "GROUP_ID"], "APP_LIST")

    logging.info("Configuration file format checking done!")

    # ------------------------------------- #
    # CONFIGURATION FILE COHERENCE CHECKING
    # ------------------------------------- #

    logging.info("Checking configuration file coherence...")

    # check topics appear once
    config_topic_name_list = [config_topic["NAME"] for config_topic in sys_conf.get("KAFKA.TOPIC_LIST")]
    if len(set(config_topic_name_list)) != len(config_topic_name_list):
        raise KafkaIotException("One or several topic(s) in topic list is(are) declared several times")

    # check group ids appear once
    config_group_id_name_list = [config_group_id["NAME"] for config_group_id in sys_conf.get("KAFKA.GROUP_ID_LIST")]
    if len(set(config_group_id_name_list)) != len(config_group_id_name_list):
        raise KafkaIotException("One or several group id(s) in group id list is(are) declared several times")

    # check applications appear once
    config_app_name_list = [config_app["NAME"] for config_app in sys_conf.get("APP_LIST")]
    if len(set(config_app_name_list)) != len(config_app_name_list):
        raise KafkaIotException("One or several app(s) in app list is(are) declared several times")

    # Check kafka rights use existing topics
    check_kafka_right_in_topic_name_list(sys_conf.get("KAFKA.KAFKA_RIGHTS_INHERITANCE"), config_topic_name_list)

    for config_group_id in sys_conf.get("KAFKA.GROUP_ID_LIST"):
Esempio n. 8
0
    def describe_topic(self, topic_name, check_topic_existence=True):
        """
        Get the information describing a topic
        :param topic_name: Name of the topic to get the information
        :type topic_name: str
        :param check_topic_existence: Check if the topic exists before trying to describe it
        :type check_topic_existence: bool
        :return: The topic information
        :rtype: Topic
        """
        if check_topic_existence:
            if topic_name not in self.list_topic_name():
                raise KafkaIotException("Topic \"%s\" doesn't exist" %
                                        topic_name)

        p = subprocess.Popen([
            self._get_topic_script(), "--describe", "--zookeeper",
            self._zookeeper_full_address, "--topic", topic_name
        ],
                             stdout=subprocess.PIPE)

        topic_partition_number = 1
        topic_replication_factor = 1
        topic_config = []
        i = 0

        while True:
            i += 1
            line = p.stdout.readline()
            if line != b'':
                line = line.decode(KafkaDriver.DEFAULT_ENCODING).replace(
                    "\n", "")
                log_console_output("DESCRIBE TOPIC", line)
                if i == 1:  # Topic info
                    matches = re.search(
                        ".*PartitionCount:\s*(\d*).*ReplicationFactor:\s*(\d*)",
                        line)
                    if len(matches.groups()
                           ) == 2:  # partition count, replication factor
                        topic_partition_number = int(matches.groups()[0])
                        topic_replication_factor = int(matches.groups()[1])
                    else:
                        raise KafkaIotException(
                            "Error when attempting to read topic information. "
                            "The received format doesn't match the required one"
                        )
                else:  # Topic config line
                    matches = re.search(
                        ".*Partition:\s*(\d*).*Leader:\s*(\d*).*Replicas:\s*(\S*).*Isr:\s*(\S*)",
                        line)
                    if len(matches.groups()
                           ) == 4:  # partition, leader, replicas, isr
                        topic_config.append(
                            TopicConfig(
                                partition=int(matches.groups()[0]),
                                leader=int(matches.groups()[1]),
                                replicas=list(
                                    int(rep)
                                    for rep in matches.groups()[2].split(",")),
                                isr=list(
                                    int(isr) for isr in matches.groups()
                                    [3].split(","))))
                    else:
                        raise KafkaIotException(
                            "Error when attempting to read topic config information. "
                            "The received format doesn't match the required one"
                        )
            else:
                break
        return Topic(topic_name, topic_replication_factor,
                     topic_partition_number, topic_config)