class KafkaAdminGroupReader:
def __init__(self, kafka_config):
self.log = logging.getLogger(__name__)
self.admin_client = KafkaAdminClient(
bootstrap_servers=kafka_config.broker_list, )
def read_group(self, groupid):
topics = set()
group_offsets = self.admin_client.list_consumer_group_offsets(groupid)
for tp in six.iterkeys(group_offsets):
topics.add(tp.topic)
return list(topics)
def read_groups(self, groupids=None, list_only=False):
if groupids is None:
groupids = self._list_groups()
if list_only:
return {groupid: [] for groupid in groupids}
groups = {}
for groupid in groupids:
topics = self.read_group(groupid)
groups[groupid] = topics
return groups
def _list_groups(self):
groups_and_protocol_types = self.admin_client.list_consumer_groups()
return [gpt[0] for gpt in groups_and_protocol_types]
def get_consumers(self, brokers):
from kafka.admin import KafkaAdminClient
adminClient = KafkaAdminClient(bootstrap_servers=brokers)
consumer_data = adminClient.list_consumer_groups()
consumers = []
for consumer in consumer_data:
consumers.append(consumer[0])
return consumers
def check_consumer_exist(bootstrap_servers, topic_name):
admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
consumer_groups = admin.describe_consumer_groups(
group_ids=[group_id for group_id, _ in admin.list_consumer_groups()])
subscriptions = {
subscription
for group in consumer_groups for member in group.members
if not isinstance(member.member_metadata, bytes)
for subscription in member.member_metadata.subscription
}
return topic_name in subscriptions
def consumer_listing(bootstrap_server):
admin_client = KafkaAdminClient(bootstrap_servers='192.0.2.11:9092')
list_consumers = []
for num in range(len(admin_client.list_consumer_groups())):
list_consumers.append(admin_client.list_consumer_groups()[num][0])
consumer_dict = {}
for num_c in range(len(list_consumers)):
consumer_dict['consumer {}'.format(num_c+1)]={}
consumer_dict['consumer {}'.format(num_c+1)]['group_id']=list_consumers[num_c]
if admin_client.describe_consumer_groups(list_consumers)[num_c][2] != 'Empty':
topic_name = admin_client.describe_consumer_groups(list_consumers)[num_c][5][0][4].decode('utf-8','ignore')
consumer_dict['consumer {}'.format(num_c+1)]['topic']=topic_name
#the pointer is made for identifying the domain and the subdomain of a topic, assuming the topic names are made with the structure of [domain].[subdomain].[schema].[version]
pointer="."
deleting_pointers=topic_name.split(pointer)
consumer_dict['consumer {}'.format(num_c+1)]['consumer_domain']=deleting_pointers[0]
consumer_dict['consumer {}'.format(num_c+1)]['consumer_subdomain']=deleting_pointers[1]
consumer_dict['consumer {}'.format(num_c+1)]['client-id']=admin_client.describe_consumer_groups(list_consumers)[num_c][5][0][1]
else:
consumer_dict['consumer {}'.format(num_c+1)]['topic']=""
consumer_dict['consumer {}'.format(num_c+1)]['client-id']=""
consumer_dict['consumer {}'.format(num_c+1)]['max_consumers']=3
return consumer_dict
from kafka.client_async import KafkaClient
from kafka import KafkaConsumer, TopicPartition
import time
from kafka.admin import KafkaAdminClient, NewTopic
consumer = KafkaConsumer(bootstrap_servers="172.31.32.39:9092", client_id='12')
print(consumer.topics()) # 获取主题列表
print(consumer.partitions_for_topic('test33333')) #获取指定主题的分区信息
adminclient = KafkaAdminClient(bootstrap_servers="172.31.32.39:9092",
client_id='12')
adminclient.list_consumer_groups()
newtopic1 = NewTopic(name='creat_test', num_partitions=2, replication_factor=1)
adminclient.create_topics([newtopic1], timeout_ms=5000)
#adminclient.create_partitions({'test33333':(2,1)})
class KafkaAdminCli(Config):
'''
KPA, kafka admin cli
create topic
delete topic
rebalance topics
'''
def __init__(self, **kwargs):
super(KafkaAdminCli, self).__init__(**kwargs)
self._topic_list = []
self._connection = None
self._topic_name = None
self._topic_partition = None
self._topic_replica = None
self._topic_assignments = None
self._topic_configs = None
# print(self.__dir__())
if self._connection is None:
if (self._ssl is None) or (self._ssl is False):
self._connection = self.kafka_admin_connect(secure='PLAINTEXT')
else:
self._connection = self.kafka_admin_connect(secure='SSL')
# print ("kwargs: {}\n, _testOnly: {} ".format(kwargs,self._testOnly.__getattribute__))
# logger.debug( "dir: %s", kwargs.__dir__)
# if self.dryrun is None:
# self._testOnly = false
# print("self._testOnly: {} ".format(self._testOnly))
def kafka_admin_connect(self, secure=None):
'''
connect kafka
'''
logger.info("connecting via KafkaAdminClient")
try:
self._connection = KafkaAdminClient(
bootstrap_servers=self.bootstrap,
client_id=self.clientId,
security_protocol=secure)
except NoBrokersAvailable as e:
print("kafka_admin_connect: {} NoBrokerAailable Error!".format(e))
logger.debug("client connected: %s", self._connection)
return self._connection
def _topic_create_from_json_file(self):
'''
create topic from topic.json file
return self.kafka_topic_create
'''
jsonData = self._open_file(self._file, "json")
# self.dup_check(jsonData)
# logging.debug('{}{}'.format(self._listTopics, self._file))
list_topic = []
for item in jsonData['topic']:
print(item)
ret = self._topic_formater_binder(item)
list_topic.append(ret)
print("list_topic: {}".format(list_topic))
#size_of_dict = len(jsonData['topic'][0].keys())
return self.kafka_topic_create(list_topic, self._testOnly)
def _topic_create_from_yaml_file(self):
'''
create topic from topic.json file
return self.kafka_topic_create
'''
jsonData = self._open_file(self._file, "yaml")
# self.dup_check(jsonData)
# logging.debug('{}{}'.format(self._listTopics, self._file))
list_topic = []
for item in jsonData['topic']:
print(item)
ret = self._topic_formater_binder(item)
list_topic.append(ret)
print("list_topic: {}".format(list_topic))
#size_of_dict = len(jsonData['topic'][0].keys())
# return self.kafka_topic_create(list_topic, self._testOnly)
def _topic_formater_binder(self, Data):
'''
:toppic_formater_binder:
topics with mixed formatted topic config to correct format
return set_binder
'''
_topic_format = {
"name": None,
"partition": None,
"replica": None,
"assignments": None,
"configs": None
}
for key, value in Data.items():
#print ("K:{}, V:{}".format(key, value))
if key == 'name':
self._topic_name = value
elif key == 'partition':
self._topic_partition = value
elif key == 'replica':
self._topic_replica = value
elif (key == 'assignments' and value != 'default'):
self._topic_assignments = value[0]
elif (key == 'configs'):
self._topic_configs = value[0]
_topic_format['name'] = self._topic_name
_topic_format['partition'] = self._topic_partition
_topic_format['replica'] = self._topic_replica
_topic_format['assignments'] = self._topic_assignments
_topic_format['configs'] = self._topic_configs
logger.debug("topic_name: %s", self._topic_name)
logger.debug("topic_partition: %s", self._topic_partition)
logger.debug("topic_replica: %s", self._topic_replica)
logger.debug("topic_replica_assignment: [%s]", self._topic_assignments)
logger.debug("topic_configuration: [%s]", self._topic_configs)
return self._newtopic_binding(_topic_format['name'],
_topic_format['partition'],
_topic_format['replica'],
_topic_format['assignments'],
_topic_format['configs'])
def kafka_topic_create(self, topic_list, flag=None):
'''
create_topics
'''
if flag is not None:
flag = self._testOnly
# logger.debug("flag: %s, _testOnly: %s", flag.__self__, self._testOnly.__self__)
try:
self._connection.create_topics(new_topics=topic_list,
validate_only=flag)
except TopicAlreadyExistsError as t:
print("Type: ", type(t), "TopicAlreadyExistsError:: \n", t)
pass
def kafka_topic_delete(self):
'''
kafka_topic_delete
'''
logger.debug("_deleteTopic:{}, _testOnly:{}, _topicNames:{}".format(
self._deleteTopics, self._testOnly, self._topicNames))
if (self._deleteTopics):
return self._topic_delete(topic_list=self._topicNames,
flag=self._testOnly)
else:
raise ("delete topic flag has not been triggerred")
def _topic_delete(self, topic_list=None, flag=None):
'''
_topic_delete
parms: topic_list (string)
'''
#topic_list=[]
logger.debug(
"topic_list:{} is going to be deleted!".format(topic_list))
if topic_list is None or len(topic_list) == 0:
raise TopicDeletionError(
"at least more than one topic need to be input")
else:
new_list = topic_list.split(",")
logger.info("newlist: %s", new_list)
if flag is not None:
flag = self._testOnly
try:
self._connection.delete_topics(new_list)
except UnknownTopicOrPartitionError as u:
logger.debug("Error while deleteing topic: {}".format(type(u)))
print("TypeL: ", type(u), "UnknownTopicOrPartitionError:: \n", u)
def kafka_list_topics(self):
logger.info("Listing Topics")
return self._connection.list_topics()
def kafka_describe_topics(self):
_topic = []
if (self._descTopics == 'all') or len(self._descTopics) == 0:
_topic = None
else:
logger.debug("type: {}, list: {}".format(type(self._descTopics),
self._descTopics))
_topic = self._descTopics.split(',')
logger.info("Describe Topics")
return self._connection.describe_topics(_topic)
def kafka_consumer_groups(self, Broker_ids=None):
#result2 = self._connection.list_consumer_groups(Broker_ids)
logger.info("Listing Consumer Groups")
return self._connection.list_consumer_groups(Broker_ids)
def _load_Json(self, data):
'''
load json data to directionary
param : json data
return: json dictionary
'''
return json.load(data)
def _open_file(self, filename, fileType):
'''
open file curor to read context
return: file
'''
with open(filename, 'rt') as rtfile:
# print (type(file))
if (fileType == "json"):
outfile = self._load_Json(rtfile)
elif (fileType == "yaml"):
outyaml = yaml.safe_load(rtfile)
logger.debug("yaml: %s", outyaml)
# print (type(outyaml))
outfile = outyaml
logger.debug("output (dictionary): %s", outfile)
return outfile
def _newtopic_binding(self,
topic_name,
topic_parition,
topic_replica,
topic_replica_assign=None,
topic_configs=None):
'''
binding newtopic with criteria of NewTopic class
required: name(string), num_partitions(int), replication_factor(int)
optional: replica_assignment(dict(int)), topic_configs(dict(str))
return: instance of class
'''
NT = NewTopic(name=topic_name,
num_partitions=topic_parition,
replication_factor=topic_replica,
replica_assignments=topic_replica_assign,
topic_configs=topic_configs)
return NT
def kafka_api_version(self):
'''
get api version
'''
return self._connection.get_api_versions()
def kafka_describe_consumer_group(self):
'''
describe consumer group
:params: consumer_group_id (string)
'''
_consumer_group_id = []
if self._descConsumerGroup is None or len(
self._descConsumerGroup) == 0:
_consumer_group_id = None
else:
_consumer_group_id = self._descConsumerGroup.split(",")
return self._connection.describe_consumer_groups(
group_ids=_consumer_group_id)
def kafka_list_consumer_group_offsets(self):
'''
list consumer group offsets
'''
if self._partitionId is None:
partition_id = None
else:
partition_id = self._partitionId.split(",")
if self._consumerGroup is None or self._consumerGroup == 'all' or self._consumerGroup == 'ALL':
consumer_group_id = self.kafka_consumer_groups()
else:
consumer_group_id = self._consumerGroup.split(",")
return self._connection.list_consumer_group_offsets(
group_id=consumer_group_id, partitions=partition_id)