def initialize_kafka_topics(self): """ Make sure the server connection is possible and the tracked topics exist """ admin_client = KafkaAdminClient(bootstrap_servers=self._servers, client_id=self._client_id) existing_topics = admin_client.list_topics() if Util.is_sequence_not_string(self._topics): non_existing_topics = set(self._topics).difference(existing_topics) if len(non_existing_topics) > 0: topic_list = [] for topic_name in non_existing_topics: # TODO: Parameterize partitions and replication factor for future scalability topic_list.append(NewTopic(name=topic_name, num_partitions=1, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False)
def create_topic(): while True: try: admin_client = KafkaAdminClient( bootstrap_servers=BOOTSTRAP_SERVERS, client_id='ws_admin') break except: logger.exception( f'Could not connect to Kafka at {BOOTSTRAP_SERVERS}') time.sleep(1) topic_list = [ NewTopic(name=OUTBOUND_TOPIC_NAME, num_partitions=1, replication_factor=1) ] logger.info(admin_client.list_topics()) admin_client.create_topics(new_topics=topic_list, validate_only=False) logger.info(f'Created topic: {OUTBOUND_TOPIC_NAME}')
def get(self): """ Get List of Topics. """ app.logger.info("Request to get list of topics.") try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) return admin.list_topics() except Exception as e: ns_topic.abort(500, str(e.args)) finally: admin.close()
def process_notification(self): bootstrap_servers = self._yaml_config.get('bootstrap_servers') read_example_topic = self._yaml_config.get('read_example_topic') write_example_topic = self._yaml_config.get('write_example_topic') admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_servers) topics = admin_client.list_topics() if read_example_topic in topics: process = Popen( args=['kafka-topics.sh', '--bootstrap-server', bootstrap_servers, '--delete', '--topic', read_example_topic, ], shell=False) print('Delete kafka topic {} status: {}'.format(read_example_topic, process.wait())) if write_example_topic in topics: process = Popen( args=['kafka-topics.sh', '--bootstrap-server', bootstrap_servers, '--delete', '--topic', write_example_topic, ], shell=False) print('Delete kafka topic {} status: {}'.format(write_example_topic, process.wait())) # Create inference online read example topic. admin_client.create_topics( new_topics=[NewTopic(name=read_example_topic, num_partitions=1, replication_factor=1)]) # Create inference vector write example topic. admin_client.create_topics( new_topics=[NewTopic(name=write_example_topic, num_partitions=1, replication_factor=1)]) self.generate_read_example()
class KafkaService(object): def __init__(self, str_bootstrap_server, log_obj): self.logger = log_obj.logger self.__str_host = str_bootstrap_server self.kafka_client = KafkaClient(hosts=self.__str_host) self.__admin = KafkaAdminClient(bootstrap_servers=self.__str_host, client_id='iot_platform') def get_topic(self, kafka_topic_name): return self.kafka_client.topics[kafka_topic_name] def consumer(self, kafka_topic_name): return self.get_topic( kafka_topic_name=kafka_topic_name).get_simple_consumer() def publisher(self, kafka_topic): return self.get_topic(kafka_topic_name=kafka_topic).get_producer() def publish(self, kafka_topic, msg): try: self.logger.info( 'Published message [MESSAGE: %s] -> kafka_topic [TOPIC: %s]' % (msg, kafka_topic)) self.publisher(kafka_topic=kafka_topic).produce(message=msg) except Exception as e: self.logger.error( 'Cannot publish message: [MESSAGE: %s] to topic: [TOPIC: %s], [ERROR: %s]' % (msg, kafka_topic, e)) def create_topic(self, list_topic_names, num_partitions=1, replication_factor=1): try: list_topics = [] topics = self.get_all_topics() for name in list_topic_names: if name in topics: return False topic = NewTopic(name=name, num_partitions=num_partitions, replication_factor=replication_factor) list_topics.append(topic) self.__admin.create_topics(new_topics=list_topics) self.logger.info('Create topics: [NAMES: %s]' % list_topic_names) except Exception as e: self.logger.error('Cannot create topic: [ERROR: %s]' % e) def delete_topic(self, topic_names): """ :param topic_names: a list of string topic names :return: """ try: self.__admin.delete_topics(topics=topic_names, timeout_ms=6000) self.logger.info('Deleted topic [TOPIC: %s]' % topic_names) except Exception as e: self.logger.error('Cannot delete topic [TOPIC: %s], [ERROR: %s]' % (topic_names, e)) def get_all_topics(self): return self.__admin.list_topics()
def runProducer(producer, id): while running: data = {"value": int(datetime.utcnow().timestamp()), "producer": id+1} producer.send(TOPIC_NAME, value=data) print(f"|{data}| sent") sleep(ITERATION_INTERVAL) pass if __name__ == "__main__": producers = [KafkaProducer(bootstrap_servers='localhost:9092', value_serializer=lambda x: dumps(x).encode('utf-8')) for i in range(PRODUCERS_QUANTITY)] admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") topics = admin_client.list_topics() if TOPIC_NAME not in topics: admin_client.create_topics([NewTopic(name=TOPIC_NAME, num_partitions=PARTITIONS_QUANTITY, replication_factor=1)], validate_only=False) with ThreadPoolExecutor(max_workers=PRODUCERS_QUANTITY) as executor: try: list( executor.map(runProducer, producers, range(len(producers)), timeout=RUNNING_TIMER_IN_SECONDS) ) except TimeoutError: print("Done executing") running = False pass pass
from kafka.admin import KafkaAdminClient, NewTopic kafka_admin_client = KafkaAdminClient(bootstrap_servers=["localhost:9092"]) try: topics = [ NewTopic(name="MCC_FILE_DATA", num_partitions=1, replication_factor=1) ] kafka_admin_client.create_topics(new_topics=topics, validate_only=False) # kafka_admin_client.delete_topics(["MCC_FILE_DATA"]) print(kafka_admin_client.list_topics()) except Exception as e: raise e finally: kafka_admin_client.close()
class KafkaAdmin: """ Администрирование кафки через библиотеку kafka_proc-python 1. Управление топиками """ """ Инициализация """ def __init__(self, hosts=None, connection_option=None, confluent=False): """ :param hosts: :param connection_option: """ if hosts is None: self.hosts = GeneralConfig.KAFKA_URL.split(',') else: self.hosts = hosts if connection_option is None: self.connection_option = default_cfg.DEFAULT_CONNECTION_OPTION_ADMIN else: self.connection_option = connection_option self.confluent = confluent """ Контекст """ def __enter__(self): """ Активация клиента админки :return: """ if self.confluent: self.admin = AdminClient( {"bootstrap.servers": ",".join(self.hosts)}) else: try: self.admin = DefaultAdminKafka(bootstrap_servers=self.hosts) except Exception as exp: # Админка конфлюента убога и не может выяснить сколько топиков raise ErrorKafkaAdmin(exp) # self.admin = AdminClient({ # "bootstrap.servers": ",".join(self.hosts) # }) # # self.confluent = True return self def __exit__(self, exc_type, exc_val, exc_tb): """ Закрытие клиента админки :param exc_type: :param exc_val: :param exc_tb: :return: """ if self.confluent: pass else: self.admin.close() """ Топики with KafkaAdmin() as ka: ka.create_topic(topics={'name': 'top_func4', 'num_partitions': 3, 'replication_factor': 3}) """ def create_topic(self, topics, topic_global_option=None): """ Создание топиков :param topics: list[dict={'name': 'top_func', 'num_partitions': 3, 'replication_factor': 3}] or simple dict class NewTopic :param topic_global_option: - create_topics add params :return: """ if self.confluent: NewTopicClass = NewTopicConf else: NewTopicClass = NewTopic topic_to_kafka = [] if isinstance(topics, list): for topic in topics: if topic.get('replication_factor') is None: topic['replication_factor'] = len(self.hosts) if self.confluent: topic['topic'] = topic.pop('name') topic_to_kafka.append(NewTopicClass(**topic)) elif isinstance(topics, dict): if topics.get('replication_factor') is None: topics['replication_factor'] = len(self.hosts) if self.confluent: topics['topic'] = topics.pop('name') topic_to_kafka.append(NewTopicClass(**topics)) else: raise ValueError('incorrect topics!') if self.confluent: result = self.admin.create_topics(topic_to_kafka) else: if topic_global_option is None: topic_global_option = { 'validate_only': False, 'timeout_ms': default_cfg.DEFAULT_BROKER_TIMEOUT_MS_OPERATIONS } result = self.admin.create_topics(new_topics=topic_to_kafka, **topic_global_option) return result def delete_topics(self, topics_name, timeout_ms=None): """ Удаление топиков. НЕ ИСПОЛЬЗОВАТЬ!!! ВНИМАНИЕ! Удаление всех топиков привело к разрушению кластера. Докер просто не смог восстановить их. Пришлось создавать заново. Предварительно ВАЖНО очистить всё, что в волюмах лежит. Кароче вообще не работает. Умирает контейнер и всё!!! Без очистки волюмов ничего не работает. :param topics_name: - список имен тем через запятую :param timeout_ms: - время ожидания ответа от брокеров :return: """ raise ValueError('THIS MAY BE DESTROY YOUR SYSTEM') if timeout_ms is None: timeout_ms = default_cfg.DEFAULT_BROKER_TIMEOUT_MS_OPERATIONS result = self.admin.delete_topics(topics=topics_name, timeout_ms=timeout_ms) return result def get_topics(self): if self.confluent: return None else: return self.admin.list_topics()
def exist_kafka_topic(topic, client_id: str, config: QueueConfig) -> bool: client = KafkaAdminClient(bootstrap_servers=config.get_url(), client_id=client_id) topics = client.list_topics() return topic in topics
def get_kafka_topics(client_id: str, config: QueueConfig): client = KafkaAdminClient(bootstrap_servers=config.get_url(), client_id=client_id) return client.list_topics()
Args: sheet_names ([list]): [It consist list of sheet names as str] """ try: topic_list = [ NewTopic(name=sheet, num_partitions=1, replication_factor=1) for sheet in sheet_names] self.kafka_admin_client.create_topics(new_topics=topic_list, validate_only=False) except errors.TopicAlreadyExistsError as e: print(e) pass except Exception as e: print("The program stopped as ", e) pass if __name__ == "__main__": dir_id = os.getenv("DIR_ID") # google drive directory id kafka_admin_client = KafkaAdminClient( bootstrap_servers=os.getenv("BOOTSTRAP_SERVERS"), client_id=os.getenv("CLIENT_ID") ) kafka_topic_creator = KafkaTopicCreator(kafka_admin_client) sheet_names = kafka_topic_creator.get_list_of_sheets(dir_id) sheet_names = space_remover(sheet_names) existing_topics = kafka_admin_client.list_topics() new_sheets = set(sheet_names) - set(existing_topics) # need to create only new topics if new_sheets.__len__() !=0 : kafka_topic_creator.create_topics(new_sheets) print("print created topics for ", new_sheets) else: print("no new sheets")
from kafka import KafkaProducer from kafka.admin import KafkaAdminClient, NewTopic admin_client = KafkaAdminClient(bootstrap_servers="172.25.0.12:9092") producer = KafkaProducer(bootstrap_servers='172.25.0.13:9092') TOPIC_NAME = 'SAMPLE_TOPIC_NAME' existing_topics = admin_client.list_topics() if TOPIC_NAME not in existing_topics: topic_list = [ NewTopic(name=TOPIC_NAME, num_partitions=1, replication_factor=1) ] admin_client.create_topics(new_topics=topic_list, validate_only=False) for i in range(1, 1001): key_bytes = bytes(str(i), encoding='utf-8') value_bytes = bytes(str(i), encoding='utf-8') producer.send(TOPIC_NAME, key=key_bytes, value=value_bytes)
class KafkaAdminCli(Config): ''' KPA, kafka admin cli create topic delete topic rebalance topics ''' def __init__(self, **kwargs): super(KafkaAdminCli, self).__init__(**kwargs) self._topic_list = [] self._connection = None self._topic_name = None self._topic_partition = None self._topic_replica = None self._topic_assignments = None self._topic_configs = None # print(self.__dir__()) if self._connection is None: if (self._ssl is None) or (self._ssl is False): self._connection = self.kafka_admin_connect(secure='PLAINTEXT') else: self._connection = self.kafka_admin_connect(secure='SSL') # print ("kwargs: {}\n, _testOnly: {} ".format(kwargs,self._testOnly.__getattribute__)) # logger.debug( "dir: %s", kwargs.__dir__) # if self.dryrun is None: # self._testOnly = false # print("self._testOnly: {} ".format(self._testOnly)) def kafka_admin_connect(self, secure=None): ''' connect kafka ''' logger.info("connecting via KafkaAdminClient") try: self._connection = KafkaAdminClient( bootstrap_servers=self.bootstrap, client_id=self.clientId, security_protocol=secure) except NoBrokersAvailable as e: print("kafka_admin_connect: {} NoBrokerAailable Error!".format(e)) logger.debug("client connected: %s", self._connection) return self._connection def _topic_create_from_json_file(self): ''' create topic from topic.json file return self.kafka_topic_create ''' jsonData = self._open_file(self._file, "json") # self.dup_check(jsonData) # logging.debug('{}{}'.format(self._listTopics, self._file)) list_topic = [] for item in jsonData['topic']: print(item) ret = self._topic_formater_binder(item) list_topic.append(ret) print("list_topic: {}".format(list_topic)) #size_of_dict = len(jsonData['topic'][0].keys()) return self.kafka_topic_create(list_topic, self._testOnly) def _topic_create_from_yaml_file(self): ''' create topic from topic.json file return self.kafka_topic_create ''' jsonData = self._open_file(self._file, "yaml") # self.dup_check(jsonData) # logging.debug('{}{}'.format(self._listTopics, self._file)) list_topic = [] for item in jsonData['topic']: print(item) ret = self._topic_formater_binder(item) list_topic.append(ret) print("list_topic: {}".format(list_topic)) #size_of_dict = len(jsonData['topic'][0].keys()) # return self.kafka_topic_create(list_topic, self._testOnly) def _topic_formater_binder(self, Data): ''' :toppic_formater_binder: topics with mixed formatted topic config to correct format return set_binder ''' _topic_format = { "name": None, "partition": None, "replica": None, "assignments": None, "configs": None } for key, value in Data.items(): #print ("K:{}, V:{}".format(key, value)) if key == 'name': self._topic_name = value elif key == 'partition': self._topic_partition = value elif key == 'replica': self._topic_replica = value elif (key == 'assignments' and value != 'default'): self._topic_assignments = value[0] elif (key == 'configs'): self._topic_configs = value[0] _topic_format['name'] = self._topic_name _topic_format['partition'] = self._topic_partition _topic_format['replica'] = self._topic_replica _topic_format['assignments'] = self._topic_assignments _topic_format['configs'] = self._topic_configs logger.debug("topic_name: %s", self._topic_name) logger.debug("topic_partition: %s", self._topic_partition) logger.debug("topic_replica: %s", self._topic_replica) logger.debug("topic_replica_assignment: [%s]", self._topic_assignments) logger.debug("topic_configuration: [%s]", self._topic_configs) return self._newtopic_binding(_topic_format['name'], _topic_format['partition'], _topic_format['replica'], _topic_format['assignments'], _topic_format['configs']) def kafka_topic_create(self, topic_list, flag=None): ''' create_topics ''' if flag is not None: flag = self._testOnly # logger.debug("flag: %s, _testOnly: %s", flag.__self__, self._testOnly.__self__) try: self._connection.create_topics(new_topics=topic_list, validate_only=flag) except TopicAlreadyExistsError as t: print("Type: ", type(t), "TopicAlreadyExistsError:: \n", t) pass def kafka_topic_delete(self): ''' kafka_topic_delete ''' logger.debug("_deleteTopic:{}, _testOnly:{}, _topicNames:{}".format( self._deleteTopics, self._testOnly, self._topicNames)) if (self._deleteTopics): return self._topic_delete(topic_list=self._topicNames, flag=self._testOnly) else: raise ("delete topic flag has not been triggerred") def _topic_delete(self, topic_list=None, flag=None): ''' _topic_delete parms: topic_list (string) ''' #topic_list=[] logger.debug( "topic_list:{} is going to be deleted!".format(topic_list)) if topic_list is None or len(topic_list) == 0: raise TopicDeletionError( "at least more than one topic need to be input") else: new_list = topic_list.split(",") logger.info("newlist: %s", new_list) if flag is not None: flag = self._testOnly try: self._connection.delete_topics(new_list) except UnknownTopicOrPartitionError as u: logger.debug("Error while deleteing topic: {}".format(type(u))) print("TypeL: ", type(u), "UnknownTopicOrPartitionError:: \n", u) def kafka_list_topics(self): logger.info("Listing Topics") return self._connection.list_topics() def kafka_describe_topics(self): _topic = [] if (self._descTopics == 'all') or len(self._descTopics) == 0: _topic = None else: logger.debug("type: {}, list: {}".format(type(self._descTopics), self._descTopics)) _topic = self._descTopics.split(',') logger.info("Describe Topics") return self._connection.describe_topics(_topic) def kafka_consumer_groups(self, Broker_ids=None): #result2 = self._connection.list_consumer_groups(Broker_ids) logger.info("Listing Consumer Groups") return self._connection.list_consumer_groups(Broker_ids) def _load_Json(self, data): ''' load json data to directionary param : json data return: json dictionary ''' return json.load(data) def _open_file(self, filename, fileType): ''' open file curor to read context return: file ''' with open(filename, 'rt') as rtfile: # print (type(file)) if (fileType == "json"): outfile = self._load_Json(rtfile) elif (fileType == "yaml"): outyaml = yaml.safe_load(rtfile) logger.debug("yaml: %s", outyaml) # print (type(outyaml)) outfile = outyaml logger.debug("output (dictionary): %s", outfile) return outfile def _newtopic_binding(self, topic_name, topic_parition, topic_replica, topic_replica_assign=None, topic_configs=None): ''' binding newtopic with criteria of NewTopic class required: name(string), num_partitions(int), replication_factor(int) optional: replica_assignment(dict(int)), topic_configs(dict(str)) return: instance of class ''' NT = NewTopic(name=topic_name, num_partitions=topic_parition, replication_factor=topic_replica, replica_assignments=topic_replica_assign, topic_configs=topic_configs) return NT def kafka_api_version(self): ''' get api version ''' return self._connection.get_api_versions() def kafka_describe_consumer_group(self): ''' describe consumer group :params: consumer_group_id (string) ''' _consumer_group_id = [] if self._descConsumerGroup is None or len( self._descConsumerGroup) == 0: _consumer_group_id = None else: _consumer_group_id = self._descConsumerGroup.split(",") return self._connection.describe_consumer_groups( group_ids=_consumer_group_id) def kafka_list_consumer_group_offsets(self): ''' list consumer group offsets ''' if self._partitionId is None: partition_id = None else: partition_id = self._partitionId.split(",") if self._consumerGroup is None or self._consumerGroup == 'all' or self._consumerGroup == 'ALL': consumer_group_id = self.kafka_consumer_groups() else: consumer_group_id = self._consumerGroup.split(",") return self._connection.list_consumer_group_offsets( group_id=consumer_group_id, partitions=partition_id)