def initialize_kafka_topics(self):
"""
Make sure the server connection is possible and the tracked topics exist
"""
admin_client = KafkaAdminClient(bootstrap_servers=self._servers, client_id=self._client_id)
existing_topics = admin_client.list_topics()
if Util.is_sequence_not_string(self._topics):
non_existing_topics = set(self._topics).difference(existing_topics)
if len(non_existing_topics) > 0:
topic_list = []
for topic_name in non_existing_topics:
# TODO: Parameterize partitions and replication factor for future scalability
topic_list.append(NewTopic(name=topic_name, num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
def create_topic():
while True:
try:
admin_client = KafkaAdminClient(
bootstrap_servers=BOOTSTRAP_SERVERS, client_id='ws_admin')
break
except:
logger.exception(
f'Could not connect to Kafka at {BOOTSTRAP_SERVERS}')
time.sleep(1)
topic_list = [
NewTopic(name=OUTBOUND_TOPIC_NAME,
num_partitions=1,
replication_factor=1)
]
logger.info(admin_client.list_topics())
admin_client.create_topics(new_topics=topic_list, validate_only=False)
logger.info(f'Created topic: {OUTBOUND_TOPIC_NAME}')
def get(self):
"""
Get List of Topics.
"""
app.logger.info("Request to get list of topics.")
try:
admin = KafkaAdminClient(
bootstrap_servers=config['cluster.broker.listeners'],
security_protocol=config['cluster.security.protocol'],
ssl_cafile=config['cluster.ssl.cafile'],
ssl_certfile=config['cluster.ssl.certfile'],
ssl_keyfile=config['cluster.ssl.keyfile'])
return admin.list_topics()
except Exception as e:
ns_topic.abort(500, str(e.args))
finally:
admin.close()
def process_notification(self):
bootstrap_servers = self._yaml_config.get('bootstrap_servers')
read_example_topic = self._yaml_config.get('read_example_topic')
write_example_topic = self._yaml_config.get('write_example_topic')
admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
topics = admin_client.list_topics()
if read_example_topic in topics:
process = Popen(
args=['kafka-topics.sh', '--bootstrap-server', bootstrap_servers, '--delete', '--topic',
read_example_topic, ], shell=False)
print('Delete kafka topic {} status: {}'.format(read_example_topic, process.wait()))
if write_example_topic in topics:
process = Popen(
args=['kafka-topics.sh', '--bootstrap-server', bootstrap_servers, '--delete', '--topic',
write_example_topic, ], shell=False)
print('Delete kafka topic {} status: {}'.format(write_example_topic, process.wait()))
# Create inference online read example topic.
admin_client.create_topics(
new_topics=[NewTopic(name=read_example_topic, num_partitions=1, replication_factor=1)])
# Create inference vector write example topic.
admin_client.create_topics(
new_topics=[NewTopic(name=write_example_topic, num_partitions=1, replication_factor=1)])
self.generate_read_example()
class KafkaService(object):
def __init__(self, str_bootstrap_server, log_obj):
self.logger = log_obj.logger
self.__str_host = str_bootstrap_server
self.kafka_client = KafkaClient(hosts=self.__str_host)
self.__admin = KafkaAdminClient(bootstrap_servers=self.__str_host,
client_id='iot_platform')
def get_topic(self, kafka_topic_name):
return self.kafka_client.topics[kafka_topic_name]
def consumer(self, kafka_topic_name):
return self.get_topic(
kafka_topic_name=kafka_topic_name).get_simple_consumer()
def publisher(self, kafka_topic):
return self.get_topic(kafka_topic_name=kafka_topic).get_producer()
def publish(self, kafka_topic, msg):
try:
self.logger.info(
'Published message [MESSAGE: %s] -> kafka_topic [TOPIC: %s]' %
(msg, kafka_topic))
self.publisher(kafka_topic=kafka_topic).produce(message=msg)
except Exception as e:
self.logger.error(
'Cannot publish message: [MESSAGE: %s] to topic: [TOPIC: %s], [ERROR: %s]'
% (msg, kafka_topic, e))
def create_topic(self,
list_topic_names,
num_partitions=1,
replication_factor=1):
try:
list_topics = []
topics = self.get_all_topics()
for name in list_topic_names:
if name in topics:
return False
topic = NewTopic(name=name,
num_partitions=num_partitions,
replication_factor=replication_factor)
list_topics.append(topic)
self.__admin.create_topics(new_topics=list_topics)
self.logger.info('Create topics: [NAMES: %s]' % list_topic_names)
except Exception as e:
self.logger.error('Cannot create topic: [ERROR: %s]' % e)
def delete_topic(self, topic_names):
"""
:param topic_names: a list of string topic names
:return:
"""
try:
self.__admin.delete_topics(topics=topic_names, timeout_ms=6000)
self.logger.info('Deleted topic [TOPIC: %s]' % topic_names)
except Exception as e:
self.logger.error('Cannot delete topic [TOPIC: %s], [ERROR: %s]' %
(topic_names, e))
def get_all_topics(self):
return self.__admin.list_topics()
def runProducer(producer, id):
while running:
data = {"value": int(datetime.utcnow().timestamp()), "producer": id+1}
producer.send(TOPIC_NAME, value=data)
print(f"|{data}| sent")
sleep(ITERATION_INTERVAL)
pass
if __name__ == "__main__":
producers = [KafkaProducer(bootstrap_servers='localhost:9092',
value_serializer=lambda x: dumps(x).encode('utf-8'))
for i in range(PRODUCERS_QUANTITY)]
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092")
topics = admin_client.list_topics()
if TOPIC_NAME not in topics:
admin_client.create_topics([NewTopic(name=TOPIC_NAME, num_partitions=PARTITIONS_QUANTITY, replication_factor=1)], validate_only=False)
with ThreadPoolExecutor(max_workers=PRODUCERS_QUANTITY) as executor:
try:
list(
executor.map(runProducer, producers, range(len(producers)),
timeout=RUNNING_TIMER_IN_SECONDS)
)
except TimeoutError:
print("Done executing")
running = False
pass
pass
from kafka.admin import KafkaAdminClient, NewTopic
kafka_admin_client = KafkaAdminClient(bootstrap_servers=["localhost:9092"])
try:
topics = [
NewTopic(name="MCC_FILE_DATA", num_partitions=1, replication_factor=1)
]
kafka_admin_client.create_topics(new_topics=topics, validate_only=False)
# kafka_admin_client.delete_topics(["MCC_FILE_DATA"])
print(kafka_admin_client.list_topics())
except Exception as e:
raise e
finally:
kafka_admin_client.close()
class KafkaAdmin:
"""
Администрирование кафки через библиотеку kafka_proc-python
1. Управление топиками
"""
"""
Инициализация
"""
def __init__(self, hosts=None, connection_option=None, confluent=False):
"""
:param hosts:
:param connection_option:
"""
if hosts is None:
self.hosts = GeneralConfig.KAFKA_URL.split(',')
else:
self.hosts = hosts
if connection_option is None:
self.connection_option = default_cfg.DEFAULT_CONNECTION_OPTION_ADMIN
else:
self.connection_option = connection_option
self.confluent = confluent
"""
Контекст
"""
def __enter__(self):
"""
Активация клиента админки
:return:
"""
if self.confluent:
self.admin = AdminClient(
{"bootstrap.servers": ",".join(self.hosts)})
else:
try:
self.admin = DefaultAdminKafka(bootstrap_servers=self.hosts)
except Exception as exp:
# Админка конфлюента убога и не может выяснить сколько топиков
raise ErrorKafkaAdmin(exp)
# self.admin = AdminClient({
# "bootstrap.servers": ",".join(self.hosts)
# })
#
# self.confluent = True
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""
Закрытие клиента админки
:param exc_type:
:param exc_val:
:param exc_tb:
:return:
"""
if self.confluent:
pass
else:
self.admin.close()
"""
Топики
with KafkaAdmin() as ka:
ka.create_topic(topics={'name': 'top_func4', 'num_partitions': 3, 'replication_factor': 3})
"""
def create_topic(self, topics, topic_global_option=None):
"""
Создание топиков
:param topics: list[dict={'name': 'top_func', 'num_partitions': 3, 'replication_factor': 3}] or simple dict
class NewTopic
:param topic_global_option: - create_topics add params
:return:
"""
if self.confluent:
NewTopicClass = NewTopicConf
else:
NewTopicClass = NewTopic
topic_to_kafka = []
if isinstance(topics, list):
for topic in topics:
if topic.get('replication_factor') is None:
topic['replication_factor'] = len(self.hosts)
if self.confluent:
topic['topic'] = topic.pop('name')
topic_to_kafka.append(NewTopicClass(**topic))
elif isinstance(topics, dict):
if topics.get('replication_factor') is None:
topics['replication_factor'] = len(self.hosts)
if self.confluent:
topics['topic'] = topics.pop('name')
topic_to_kafka.append(NewTopicClass(**topics))
else:
raise ValueError('incorrect topics!')
if self.confluent:
result = self.admin.create_topics(topic_to_kafka)
else:
if topic_global_option is None:
topic_global_option = {
'validate_only': False,
'timeout_ms':
default_cfg.DEFAULT_BROKER_TIMEOUT_MS_OPERATIONS
}
result = self.admin.create_topics(new_topics=topic_to_kafka,
**topic_global_option)
return result
def delete_topics(self, topics_name, timeout_ms=None):
"""
Удаление топиков. НЕ ИСПОЛЬЗОВАТЬ!!!
ВНИМАНИЕ! Удаление всех топиков привело к разрушению кластера.
Докер просто не смог восстановить их. Пришлось создавать заново.
Предварительно ВАЖНО очистить всё, что в волюмах лежит.
Кароче вообще не работает. Умирает контейнер и всё!!!
Без очистки волюмов ничего не работает.
:param topics_name: - список имен тем через запятую
:param timeout_ms: - время ожидания ответа от брокеров
:return:
"""
raise ValueError('THIS MAY BE DESTROY YOUR SYSTEM')
if timeout_ms is None:
timeout_ms = default_cfg.DEFAULT_BROKER_TIMEOUT_MS_OPERATIONS
result = self.admin.delete_topics(topics=topics_name,
timeout_ms=timeout_ms)
return result
def get_topics(self):
if self.confluent:
return None
else:
return self.admin.list_topics()
def exist_kafka_topic(topic, client_id: str, config: QueueConfig) -> bool:
client = KafkaAdminClient(bootstrap_servers=config.get_url(),
client_id=client_id)
topics = client.list_topics()
return topic in topics
def get_kafka_topics(client_id: str, config: QueueConfig):
client = KafkaAdminClient(bootstrap_servers=config.get_url(),
client_id=client_id)
return client.list_topics()
Args:
sheet_names ([list]): [It consist list of sheet names as str]
"""
try:
topic_list = [ NewTopic(name=sheet, num_partitions=1, replication_factor=1) for sheet in sheet_names]
self.kafka_admin_client.create_topics(new_topics=topic_list, validate_only=False)
except errors.TopicAlreadyExistsError as e:
print(e)
pass
except Exception as e:
print("The program stopped as ", e)
pass
if __name__ == "__main__":
dir_id = os.getenv("DIR_ID") # google drive directory id
kafka_admin_client = KafkaAdminClient(
bootstrap_servers=os.getenv("BOOTSTRAP_SERVERS"),
client_id=os.getenv("CLIENT_ID")
)
kafka_topic_creator = KafkaTopicCreator(kafka_admin_client)
sheet_names = kafka_topic_creator.get_list_of_sheets(dir_id)
sheet_names = space_remover(sheet_names)
existing_topics = kafka_admin_client.list_topics()
new_sheets = set(sheet_names) - set(existing_topics) # need to create only new topics
if new_sheets.__len__() !=0 :
kafka_topic_creator.create_topics(new_sheets)
print("print created topics for ", new_sheets)
else:
print("no new sheets")
from kafka import KafkaProducer
from kafka.admin import KafkaAdminClient, NewTopic
admin_client = KafkaAdminClient(bootstrap_servers="172.25.0.12:9092")
producer = KafkaProducer(bootstrap_servers='172.25.0.13:9092')
TOPIC_NAME = 'SAMPLE_TOPIC_NAME'
existing_topics = admin_client.list_topics()
if TOPIC_NAME not in existing_topics:
topic_list = [
NewTopic(name=TOPIC_NAME, num_partitions=1, replication_factor=1)
]
admin_client.create_topics(new_topics=topic_list, validate_only=False)
for i in range(1, 1001):
key_bytes = bytes(str(i), encoding='utf-8')
value_bytes = bytes(str(i), encoding='utf-8')
producer.send(TOPIC_NAME, key=key_bytes, value=value_bytes)
class KafkaAdminCli(Config):
'''
KPA, kafka admin cli
create topic
delete topic
rebalance topics
'''
def __init__(self, **kwargs):
super(KafkaAdminCli, self).__init__(**kwargs)
self._topic_list = []
self._connection = None
self._topic_name = None
self._topic_partition = None
self._topic_replica = None
self._topic_assignments = None
self._topic_configs = None
# print(self.__dir__())
if self._connection is None:
if (self._ssl is None) or (self._ssl is False):
self._connection = self.kafka_admin_connect(secure='PLAINTEXT')
else:
self._connection = self.kafka_admin_connect(secure='SSL')
# print ("kwargs: {}\n, _testOnly: {} ".format(kwargs,self._testOnly.__getattribute__))
# logger.debug( "dir: %s", kwargs.__dir__)
# if self.dryrun is None:
# self._testOnly = false
# print("self._testOnly: {} ".format(self._testOnly))
def kafka_admin_connect(self, secure=None):
'''
connect kafka
'''
logger.info("connecting via KafkaAdminClient")
try:
self._connection = KafkaAdminClient(
bootstrap_servers=self.bootstrap,
client_id=self.clientId,
security_protocol=secure)
except NoBrokersAvailable as e:
print("kafka_admin_connect: {} NoBrokerAailable Error!".format(e))
logger.debug("client connected: %s", self._connection)
return self._connection
def _topic_create_from_json_file(self):
'''
create topic from topic.json file
return self.kafka_topic_create
'''
jsonData = self._open_file(self._file, "json")
# self.dup_check(jsonData)
# logging.debug('{}{}'.format(self._listTopics, self._file))
list_topic = []
for item in jsonData['topic']:
print(item)
ret = self._topic_formater_binder(item)
list_topic.append(ret)
print("list_topic: {}".format(list_topic))
#size_of_dict = len(jsonData['topic'][0].keys())
return self.kafka_topic_create(list_topic, self._testOnly)
def _topic_create_from_yaml_file(self):
'''
create topic from topic.json file
return self.kafka_topic_create
'''
jsonData = self._open_file(self._file, "yaml")
# self.dup_check(jsonData)
# logging.debug('{}{}'.format(self._listTopics, self._file))
list_topic = []
for item in jsonData['topic']:
print(item)
ret = self._topic_formater_binder(item)
list_topic.append(ret)
print("list_topic: {}".format(list_topic))
#size_of_dict = len(jsonData['topic'][0].keys())
# return self.kafka_topic_create(list_topic, self._testOnly)
def _topic_formater_binder(self, Data):
'''
:toppic_formater_binder:
topics with mixed formatted topic config to correct format
return set_binder
'''
_topic_format = {
"name": None,
"partition": None,
"replica": None,
"assignments": None,
"configs": None
}
for key, value in Data.items():
#print ("K:{}, V:{}".format(key, value))
if key == 'name':
self._topic_name = value
elif key == 'partition':
self._topic_partition = value
elif key == 'replica':
self._topic_replica = value
elif (key == 'assignments' and value != 'default'):
self._topic_assignments = value[0]
elif (key == 'configs'):
self._topic_configs = value[0]
_topic_format['name'] = self._topic_name
_topic_format['partition'] = self._topic_partition
_topic_format['replica'] = self._topic_replica
_topic_format['assignments'] = self._topic_assignments
_topic_format['configs'] = self._topic_configs
logger.debug("topic_name: %s", self._topic_name)
logger.debug("topic_partition: %s", self._topic_partition)
logger.debug("topic_replica: %s", self._topic_replica)
logger.debug("topic_replica_assignment: [%s]", self._topic_assignments)
logger.debug("topic_configuration: [%s]", self._topic_configs)
return self._newtopic_binding(_topic_format['name'],
_topic_format['partition'],
_topic_format['replica'],
_topic_format['assignments'],
_topic_format['configs'])
def kafka_topic_create(self, topic_list, flag=None):
'''
create_topics
'''
if flag is not None:
flag = self._testOnly
# logger.debug("flag: %s, _testOnly: %s", flag.__self__, self._testOnly.__self__)
try:
self._connection.create_topics(new_topics=topic_list,
validate_only=flag)
except TopicAlreadyExistsError as t:
print("Type: ", type(t), "TopicAlreadyExistsError:: \n", t)
pass
def kafka_topic_delete(self):
'''
kafka_topic_delete
'''
logger.debug("_deleteTopic:{}, _testOnly:{}, _topicNames:{}".format(
self._deleteTopics, self._testOnly, self._topicNames))
if (self._deleteTopics):
return self._topic_delete(topic_list=self._topicNames,
flag=self._testOnly)
else:
raise ("delete topic flag has not been triggerred")
def _topic_delete(self, topic_list=None, flag=None):
'''
_topic_delete
parms: topic_list (string)
'''
#topic_list=[]
logger.debug(
"topic_list:{} is going to be deleted!".format(topic_list))
if topic_list is None or len(topic_list) == 0:
raise TopicDeletionError(
"at least more than one topic need to be input")
else:
new_list = topic_list.split(",")
logger.info("newlist: %s", new_list)
if flag is not None:
flag = self._testOnly
try:
self._connection.delete_topics(new_list)
except UnknownTopicOrPartitionError as u:
logger.debug("Error while deleteing topic: {}".format(type(u)))
print("TypeL: ", type(u), "UnknownTopicOrPartitionError:: \n", u)
def kafka_list_topics(self):
logger.info("Listing Topics")
return self._connection.list_topics()
def kafka_describe_topics(self):
_topic = []
if (self._descTopics == 'all') or len(self._descTopics) == 0:
_topic = None
else:
logger.debug("type: {}, list: {}".format(type(self._descTopics),
self._descTopics))
_topic = self._descTopics.split(',')
logger.info("Describe Topics")
return self._connection.describe_topics(_topic)
def kafka_consumer_groups(self, Broker_ids=None):
#result2 = self._connection.list_consumer_groups(Broker_ids)
logger.info("Listing Consumer Groups")
return self._connection.list_consumer_groups(Broker_ids)
def _load_Json(self, data):
'''
load json data to directionary
param : json data
return: json dictionary
'''
return json.load(data)
def _open_file(self, filename, fileType):
'''
open file curor to read context
return: file
'''
with open(filename, 'rt') as rtfile:
# print (type(file))
if (fileType == "json"):
outfile = self._load_Json(rtfile)
elif (fileType == "yaml"):
outyaml = yaml.safe_load(rtfile)
logger.debug("yaml: %s", outyaml)
# print (type(outyaml))
outfile = outyaml
logger.debug("output (dictionary): %s", outfile)
return outfile
def _newtopic_binding(self,
topic_name,
topic_parition,
topic_replica,
topic_replica_assign=None,
topic_configs=None):
'''
binding newtopic with criteria of NewTopic class
required: name(string), num_partitions(int), replication_factor(int)
optional: replica_assignment(dict(int)), topic_configs(dict(str))
return: instance of class
'''
NT = NewTopic(name=topic_name,
num_partitions=topic_parition,
replication_factor=topic_replica,
replica_assignments=topic_replica_assign,
topic_configs=topic_configs)
return NT
def kafka_api_version(self):
'''
get api version
'''
return self._connection.get_api_versions()
def kafka_describe_consumer_group(self):
'''
describe consumer group
:params: consumer_group_id (string)
'''
_consumer_group_id = []
if self._descConsumerGroup is None or len(
self._descConsumerGroup) == 0:
_consumer_group_id = None
else:
_consumer_group_id = self._descConsumerGroup.split(",")
return self._connection.describe_consumer_groups(
group_ids=_consumer_group_id)
def kafka_list_consumer_group_offsets(self):
'''
list consumer group offsets
'''
if self._partitionId is None:
partition_id = None
else:
partition_id = self._partitionId.split(",")
if self._consumerGroup is None or self._consumerGroup == 'all' or self._consumerGroup == 'ALL':
consumer_group_id = self.kafka_consumer_groups()
else:
consumer_group_id = self._consumerGroup.split(",")
return self._connection.list_consumer_group_offsets(
group_id=consumer_group_id, partitions=partition_id)