def get(self, search_string=None):
"""
Get a config value using a search string
:param search_string: Config param separated with a dot '.'
:type search_string: str
:return: The value of the config element if it exists, raise an exception otherwise
"""
if search_string is None:
return self._config_content
if isinstance(search_string, str):
param_array = search_string.split(".")
performed_param_str = None
ret = self._config_content
for param in param_array:
performed_param_str = performed_param_str + "." + param if performed_param_str is not None else param
if param in ret:
ret = ret[param]
else:
raise KafkaIotException("Can't find param %s in config file" % performed_param_str)
return ret
else:
raise KafkaIotException("Given parameter for search_string must be a string! %s given instead" %
str(type(search_string)))
def delete_topic(self, topic_name, check_topic_existence=True):
"""
Attempt to delete a given topic.
This will have no effect if delete.topic.enable is not set to true in kafka config
:param topic_name: The name of the topic to delete
:type topic_name: str
:param check_topic_existence: Check if the topic exists before trying to delete it
:type check_topic_existence: bool
"""
if check_topic_existence:
if topic_name not in self.list_topic_name():
raise KafkaIotException("Topic \"%s\" doesn't exist" %
topic_name)
p = subprocess.Popen([
self._get_topic_script(), "--delete", "--zookeeper",
self._zookeeper_full_address, "--topic", topic_name
],
stdout=subprocess.PIPE)
while True:
line = p.stdout.readline()
if line != b'':
line = line.decode(KafkaDriver.DEFAULT_ENCODING).replace(
"\n", "")
log_console_output("DELETE TOPIC", line)
else:
break
def check_kafka_right_in_topic_name_list(kafka_rights, topic_name_list):
"""
Check kafka right topics exists in topic list
:param kafka_rights: Dictionary with read and write topic list
:type kafka_rights: dict
:param topic_name_list: List of topic name
:type topic_name_list: list[str]
:return: True if all the topics in kafka rights are in topic name list. Raise an exception otherwise
"""
for read_topic in kafka_rights["READ"]:
if read_topic != "*" and read_topic not in topic_name_list:
raise KafkaIotException(
"READ topic \"%s\" is not declared in topic list" % read_topic)
for write_topic in kafka_rights["WRITE"]:
if write_topic != "*" and write_topic not in topic_name_list:
raise KafkaIotException(
"WRITE topic \"%s\" is not declared in topic list" %
write_topic)
def alter_topic(self,
topic_name,
partition_number=-1,
check_topic_existence=True):
"""
Alter a topic replication factor and / or partition number
:param topic_name: Name of the topic to alter
:type topic_name: str
:param partition_number: New partition number to be applied. No effect when set to -1
:type partition_number: int
:param check_topic_existence: Check if the topic exists before trying to delete it
:type check_topic_existence: bool
"""
if check_topic_existence:
if topic_name not in self.list_topic_name():
raise KafkaIotException("Topic \"%s\" doesn't exist" %
topic_name)
topic = self.describe_topic(topic_name, False)
if topic.partition_number != partition_number > 0:
if partition_number < topic.partition_number: # If the number of partition has te be decreased
self.delete_topic(topic_name, False)
self.create_topic(topic_name, topic.replication_factor,
partition_number)
logging.warning(
"Topic \"%s\" has been deleted and re-created in order to reduce its partition number "
"from %d to %d" %
(topic_name, topic.partition_number, partition_number))
else: # Increase the number of partition
p = subprocess.Popen([
self._get_topic_script(), "--alter", "--zookeeper",
self._zookeeper_full_address, "--topic", topic_name,
"--partitions",
str(partition_number)
],
stdout=subprocess.PIPE)
while True:
line = p.stdout.readline()
if line != b'':
line = line.decode(
KafkaDriver.DEFAULT_ENCODING).replace("\n", "")
log_console_output("ALTER TOPIC PARTITION", line)
else:
break
def init_logger(log_level, log_location, app_name):
# Getting log level
log_level = _define_log_level(log_level)
# Modify logger log level
logger = logging.getLogger()
logger.setLevel(log_level)
# Set file formatter
formatter = logging.Formatter("%(asctime)s :: %(levelname)s :: " +
app_name + " :: %(message)s")
log_filename = "%s_log.txt" % app_name
log_location = expand_var_and_user(log_location)
if os.path.isdir(log_location):
log_file_path = os.path.join(log_location, log_filename)
need_roll = os.path.isfile(log_file_path)
# Redirect logs into a log file
file_handler = RotatingFileHandler(log_file_path,
backupCount=10,
maxBytes=2 * 1024 * 1024)
file_handler.setLevel(log_level)
file_handler.setFormatter(formatter)
if need_roll:
file_handler.doRollover()
logger.addHandler(file_handler)
# Redirect logs into the user console
stream_handler = logging.StreamHandler()
stream_handler.setLevel(log_level)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
else:
raise KafkaIotException(
"Log directory: %s does not exist, create it before relaunching the program"
% log_location)
def create_topic(self,
topic_name,
replication_factor,
partitions,
check_topic_existence=True):
"""
Attempt to create a topic
:param topic_name: The name of the topic to be created
:type topic_name: str
:param replication_factor: The replication factor of the topic
:type replication_factor: int
:param partitions: The number of partition of the topic
:type partitions: int
:param check_topic_existence: Check if the topic exists before trying to create it
"""
if check_topic_existence:
if topic_name in self.list_topic_name():
raise KafkaIotException("Topic \"%s\" already exists" %
topic_name)
p = subprocess.Popen([
self._get_topic_script(), "--create", "--zookeeper",
self._zookeeper_full_address, "--replication-factor",
str(replication_factor), "--partitions",
str(partitions), "--topic", topic_name
],
stdout=subprocess.PIPE)
while True:
line = p.stdout.readline()
if line != b'':
line = line.decode(KafkaDriver.DEFAULT_ENCODING).replace(
"\n", "")
log_console_output("CREATE TOPIC", line)
else:
break
for app in sys_conf.get("APP_LIST"):
check_fields_in_dict(app, ["NAME", "GROUP_ID"], "APP_LIST")
logging.info("Configuration file format checking done!")
# ------------------------------------- #
# CONFIGURATION FILE COHERENCE CHECKING
# ------------------------------------- #
logging.info("Checking configuration file coherence...")
# check topics appear once
config_topic_name_list = [config_topic["NAME"] for config_topic in sys_conf.get("KAFKA.TOPIC_LIST")]
if len(set(config_topic_name_list)) != len(config_topic_name_list):
raise KafkaIotException("One or several topic(s) in topic list is(are) declared several times")
# check group ids appear once
config_group_id_name_list = [config_group_id["NAME"] for config_group_id in sys_conf.get("KAFKA.GROUP_ID_LIST")]
if len(set(config_group_id_name_list)) != len(config_group_id_name_list):
raise KafkaIotException("One or several group id(s) in group id list is(are) declared several times")
# check applications appear once
config_app_name_list = [config_app["NAME"] for config_app in sys_conf.get("APP_LIST")]
if len(set(config_app_name_list)) != len(config_app_name_list):
raise KafkaIotException("One or several app(s) in app list is(are) declared several times")
# Check kafka rights use existing topics
check_kafka_right_in_topic_name_list(sys_conf.get("KAFKA.KAFKA_RIGHTS_INHERITANCE"), config_topic_name_list)
for config_group_id in sys_conf.get("KAFKA.GROUP_ID_LIST"):
def describe_topic(self, topic_name, check_topic_existence=True):
"""
Get the information describing a topic
:param topic_name: Name of the topic to get the information
:type topic_name: str
:param check_topic_existence: Check if the topic exists before trying to describe it
:type check_topic_existence: bool
:return: The topic information
:rtype: Topic
"""
if check_topic_existence:
if topic_name not in self.list_topic_name():
raise KafkaIotException("Topic \"%s\" doesn't exist" %
topic_name)
p = subprocess.Popen([
self._get_topic_script(), "--describe", "--zookeeper",
self._zookeeper_full_address, "--topic", topic_name
],
stdout=subprocess.PIPE)
topic_partition_number = 1
topic_replication_factor = 1
topic_config = []
i = 0
while True:
i += 1
line = p.stdout.readline()
if line != b'':
line = line.decode(KafkaDriver.DEFAULT_ENCODING).replace(
"\n", "")
log_console_output("DESCRIBE TOPIC", line)
if i == 1: # Topic info
matches = re.search(
".*PartitionCount:\s*(\d*).*ReplicationFactor:\s*(\d*)",
line)
if len(matches.groups()
) == 2: # partition count, replication factor
topic_partition_number = int(matches.groups()[0])
topic_replication_factor = int(matches.groups()[1])
else:
raise KafkaIotException(
"Error when attempting to read topic information. "
"The received format doesn't match the required one"
)
else: # Topic config line
matches = re.search(
".*Partition:\s*(\d*).*Leader:\s*(\d*).*Replicas:\s*(\S*).*Isr:\s*(\S*)",
line)
if len(matches.groups()
) == 4: # partition, leader, replicas, isr
topic_config.append(
TopicConfig(
partition=int(matches.groups()[0]),
leader=int(matches.groups()[1]),
replicas=list(
int(rep)
for rep in matches.groups()[2].split(",")),
isr=list(
int(isr) for isr in matches.groups()
[3].split(","))))
else:
raise KafkaIotException(
"Error when attempting to read topic config information. "
"The received format doesn't match the required one"
)
else:
break
return Topic(topic_name, topic_replication_factor,
topic_partition_number, topic_config)