def create_topic(self, topic_name: str, num_partitions: int = 1, replication_factor: int = 1): """ Creates a new topic if not already present :param topic_name: New topic name :param num_partitions: Number of partitions. Default value is 1. :param replication_factor: Replication factor. Default value is 1. """ logger.debug(f"Creating a topic called {topic_name}") admin = KafkaAdminClient(**self.settings) existing_topics = admin.list_topics() if topic_name in existing_topics: admin.close() return topic = [ NewTopic(topic_name, num_partitions=num_partitions, replication_factor=replication_factor) ] admin.create_topics(topic) admin.close() logger.debug(f"Topic {topic_name} created")
def run_kafka_server(): # TODO get the json file path input_file = "police-department-calls-for-service.json" # same folder as this script # TODO fill in blanks # GK Note: I have completed the code template provided by Udacity. # The provided code implies that the topic is autogenerated by KafkaProducer # The provided code doesn't call the Kafka AdminClient to create the topic explicitly # To experiment with number of Kafka partitions for performance, I create an admin client bootstrap_servers = 'localhost:9092' my_topic = "sf.police.calls" admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_servers) try: print("attempting to create new topic") topic_list = [] nb_partitions = 2 topic_list.append( NewTopic(name=my_topic, num_partitions=nb_partitions, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False) except Exception: print("topic already exists") producer = producer_server.ProducerServer( #producer = producer_server.ProducerServer( input_file=input_file, # crime events file topic=my_topic, # freely chosen bootstrap_servers=bootstrap_servers, # standard choice for Kafka server client_id="producer-01" # ID chosen for this producer ) return producer
def create_topic(brokers, topic, partition_count=1, replica_count=1): """Create a topic if it does not exist. Args: brokers (list): The 'host[:port]' list that the producer should contact to bootstrap initial cluster metadata. topic (str): Topic where the message will be published. partition_count (int): Specified partition number (default 1). replica_count (int): Specified replication factor (default 1). Returns: partitions (set): A set including partition number. """ consumer = KafkaConsumer(bootstrap_servers=brokers) topics = consumer.topics() if topic in topics: partitions = consumer.partitions_for_topic(topic) consumer.close() else: consumer.close() admin = KafkaAdminClient(bootstrap_servers=brokers) admin.create_topics([ NewTopic( name=topic, num_partitions=partition_count, replication_factor=replica_count, ), ]) admin.close() partitions = set([p for p in range(partition_count)]) return partitions
def create_topic(): global topic # Create the kafka topic tries = 30 exit = False while not exit: try: try: topic = admin.NewTopic(name="slice", num_partitions=1, replication_factor=1) broker = KafkaAdminClient(bootstrap_servers="kafka:19092") broker.create_topics([topic]) except errors.TopicAlreadyExistsError: logger.warning("Topic exists already") else: logger.info("New topic") except errors.NoBrokersAvailable as KafkaError: if tries > 0: tries -= 1 logger.warning( "Kafka not ready yet. Tries remaining: {0}".format(tries)) time.sleep(5) else: logger.error(KafkaError) else: exit = True tries = 30
def main(): # Create 'my-topic' Kafka topic try: admin = KafkaAdminClient(bootstrap_servers="localhost:9092") topic = NewTopic(name="my-topic", num_partitions=1, replication_factor=1) admin.create_topics([topic]) except Exception: pass tasks = [Producer(), Consumer()] # Start threads of a publisher/producer and a subscriber/consumer to 'my-topic' Kafka topic for t in tasks: t.start() time.sleep(10) # Stop threads for task in tasks: task.stop() for task in tasks: task.join()
def _init_topic(self): client = KafkaAdminClient(bootstrap_servers=self.servers) try: client.create_topics( [NewTopic(self.topic, num_partitions=len(self.databases), replication_factor=1)] ) except kafka.errors.TopicAlreadyExistsError: pass
def main(event, context): print("Looking up broker ARN") svc_client = boto3.client('servicediscovery') response = svc_client.discover_instances(NamespaceName=NAMESPACE, ServiceName=SERVICE) broker_arn = response['Instances'][0]['Attributes']['broker_arn'] print("Got broker ARN {0}".format(broker_arn)) print("Looking up broker string") msk_client = boto3.client('kafka') response = msk_client.get_bootstrap_brokers(ClusterArn=broker_arn) broker_string = response['BootstrapBrokerStringTls'] print("Got broker string {0}".format(broker_string)) # make sure topic exists print("Checking if topic {0} exists".format(TOPIC_NAME)) kclient = KafkaConsumer(bootstrap_servers=broker_string, security_protocol='SSL') existing_topics = kclient.topics() if TOPIC_NAME in existing_topics: print("Topic {0} exists".format(TOPIC_NAME)) else: print("Topic {0} does not exist, creating".format(TOPIC_NAME)) topic_list = [] topic = NewTopic(name=TOPIC_NAME, num_partitions=1, replication_factor=1) topic_list.append(topic) kadmin = KafkaAdminClient(bootstrap_servers=broker_string, security_protocol='SSL') kadmin.create_topics(new_topics=topic_list) kadmin.close() kclient.close() producer = KafkaProducer(bootstrap_servers=broker_string, security_protocol='SSL') while True: remaining_time_millis = context.get_remaining_time_in_millis() if remaining_time_millis < MIN_TIME_REMAINING_MILLIS: print("Time left ({0}) is less than time required ({1}), exiting". format(str(remaining_time_millis), str(MIN_TIME_REMAINING_MILLIS))) break else: print( "Time left ({0}) is greater than time required ({1}), sending". format(str(remaining_time_millis), str(MIN_TIME_REMAINING_MILLIS))) msg = "Kafka message sent at {0}".format(str(time.time())) producer.send(TOPIC_NAME, msg.encode('utf-8')) producer.flush() time.sleep(SLEEP_SECONDS) producer.close() print("All done")
def test_kafka_virtual_columns2(kafka_cluster): admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") topic_list = [] topic_list.append(NewTopic(name="virt2_0", num_partitions=2, replication_factor=1)) topic_list.append(NewTopic(name="virt2_1", num_partitions=2, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False) instance.query(''' CREATE TABLE test.kafka (value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', kafka_topic_list = 'virt2_0,virt2_1', kafka_group_name = 'virt2', kafka_format = 'JSONEachRow'; CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT value, _key, _topic, _partition, _offset, toUnixTimestamp(_timestamp) FROM test.kafka; ''') producer = KafkaProducer(bootstrap_servers="localhost:9092") producer.send(topic='virt2_0', value=json.dumps({'value': 1}), partition=0, key='k1', timestamp_ms=1577836801000) producer.send(topic='virt2_0', value=json.dumps({'value': 2}), partition=0, key='k2', timestamp_ms=1577836802000) producer.flush() time.sleep(1) producer.send(topic='virt2_0', value=json.dumps({'value': 3}), partition=1, key='k3', timestamp_ms=1577836803000) producer.send(topic='virt2_0', value=json.dumps({'value': 4}), partition=1, key='k4', timestamp_ms=1577836804000) producer.flush() time.sleep(1) producer.send(topic='virt2_1', value=json.dumps({'value': 5}), partition=0, key='k5', timestamp_ms=1577836805000) producer.send(topic='virt2_1', value=json.dumps({'value': 6}), partition=0, key='k6', timestamp_ms=1577836806000) producer.flush() time.sleep(1) producer.send(topic='virt2_1', value=json.dumps({'value': 7}), partition=1, key='k7', timestamp_ms=1577836807000) producer.send(topic='virt2_1', value=json.dumps({'value': 8}), partition=1, key='k8', timestamp_ms=1577836808000) producer.flush() time.sleep(10) result = instance.query("SELECT * FROM test.view ORDER BY value", ignore_error=True) expected = '''\ 1 k1 virt2_0 0 0 1577836801 2 k2 virt2_0 0 1 1577836802 3 k3 virt2_0 1 0 1577836803 4 k4 virt2_0 1 1 1577836804 5 k5 virt2_1 0 0 1577836805 6 k6 virt2_1 0 1 1577836806 7 k7 virt2_1 1 0 1577836807 8 k8 virt2_1 1 1 1577836808 ''' assert TSV(result) == TSV(expected)
def create(topicname): from kafka import KafkaAdminClient from kafka.admin import NewTopic kac = KafkaAdminClient(bootstrap_servers='localhost:9092') kac.create_topics(new_topics=[ NewTopic(name=topicname, num_partitions=1, replication_factor=1) ]) info_message = 'Created ' + topicname print(info_message) return info_message
class Admin: def __init__(self): self.client = KafkaAdminClient(bootstrap_servers=app.config['KAFKA_URL']) def create_topics(self, topics): topics = [NewTopic(topic, 1, 1) for topic in topics] self.client.create_topics(new_topics=topics) def delete_topics(self, topics): self.client.delete_topics(topics=topics)
def add_topic(args): admin = KafkaAdminClient(bootstrap_servers=[args.broker]) admin.create_topics([ NewTopic( name=args.topic, num_partitions=args.partitions, replication_factor=args.replication_factor, ), ]) admin.close()
def test_connectivity(): admin = KafkaAdminClient(bootstrap_servers=KAFKA_BROKERS, security_protocol='SSL') try: admin.create_topics([NewTopic(TOPIC_NAME, 3, 2)]) except TopicAlreadyExistsError: print(f"Topic '{TOPIC_NAME}' already exists.") producer = KafkaProducer(bootstrap_servers=KAFKA_BROKERS, security_protocol='SSL') producer.send(topic=TOPIC_NAME, value=b'yes')
class AdminClient(KafkaPython): def __init__(self, bootstrap_servers=None, **kwargs): super().__init__(servers=bootstrap_servers) admin_client_config.update(kwargs) self.engine = KafkaAdminClient( bootstrap_servers=self._bootstrap_servers, client_id=self._client_id, request_timeout_ms=self._request_timeout_ms, **admin_client_config) def create_topics(self, topic_list: list): new_topic = [] for k, v in enumerate(topic_list): new_topic.append( NewTopics(name=v['name'], num_partitions=v['num_partitions'], replication_factor=v['replication_factor'], replica_assignments=v['replica_assignments'], topic_configs=v['topic_configs'])) if not Consumer().get_user_topics().intersection( {item['name'] for i, item in enumerate(topic_list)}): try: self.engine.create_topics(new_topic, **create_topic_config) except KafkaError as e: _logger.error(e) self.engine.close() else: _logger.error( self._logMsg(create_topic_fail_code, self._client_id, 'topic重复')) return def delete_topics(self, topic: list): if Consumer().get_user_topics().intersection(set(topic)): try: self.engine.delete_topics(topic, self._request_timeout_ms) except KafkaError as e: _logger.error( self._logMsg(delete_topic_fail_code, self._client_id, '删除的topic失败:%s' % e)) self.engine.close() else: _logger.error( self._logMsg(delete_topic_fail_code, self._client_id, '需删除的topic不存在')) return
def createTopic(self, bootstrapServer, topic, numPartitions, replicationFactor): try: admin = KafkaAdminClient(bootstrapServer) topic = NewTopic(name=topic, num_partitions=numPartitions, replication_factor=replicationFactor) admin.create_topics([topic]) except Exception as e: logger.error('Error to create topic %s' % e) pass
def ensure_kafka_topic_exists(config): set_up = False for i in range(1, 12): try: admin_client = KafkaAdminClient(bootstrap_servers=config.brokers) admin_client.create_topics([NewTopic(config.topic, 1, 1)]) set_up = True break except (UnrecognizedBrokerVersion, ValueError): time.sleep(5) if not set_up: raise Exception('Failed it setup kafka topic')
def main(): # Create 'my-topic' Kafka topic try: admin = KafkaAdminClient(bootstrap_servers='localhost:9092') topic = NewTopic(name='outputframes', num_partitions=1, replication_factor=1) admin.create_topics([topic]) except Exception: pass subscribe()
def _shedual_task(self): from confluent_kafka import Consumer as ConfluentConsumer # 这个包不好安装,用户用这个中间件的时候自己再想办法安装。 try: admin_client = KafkaAdminClient( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) admin_client.create_topics([NewTopic(self._queue_name, 10, 1)]) # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)}) except TopicAlreadyExistsError: pass self._producer = KafkaProducer( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) # consumer 配置 https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md self._confluent_consumer = ConfluentConsumer({ 'bootstrap.servers': ','.join(frame_config.KAFKA_BOOTSTRAP_SERVERS), 'group.id': 'frame_group', 'auto.offset.reset': 'earliest', 'enable.auto.commit': False }) self._confluent_consumer.subscribe([self._queue_name]) self._recent_commit_time = time.time() self._partion__offset_consume_status_map = defaultdict(OrderedDict) while 1: msg = self._confluent_consumer.poll(timeout=10) self._manually_commit() if msg is None: continue if msg.error(): print("Consumer error: {}".format(msg.error())) continue # msg的类型 https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#message # value() offset() partition() # print('Received message: {}'.format(msg.value().decode('utf-8'))) # noqa self._partion__offset_consume_status_map[msg.partition()][ msg.offset()] = 0 kw = { 'partition': msg.partition(), 'offset': msg.offset(), 'body': json.loads(msg.value()) } # noqa if self._is_show_message_get_from_broker: self.logger.debug( f'从kafka的 [{self._queue_name}] 主题,分区 {msg.partition()} 中 的 offset {msg.offset()} 取出的消息是: {msg.value()}' ) # noqa self._submit_task(kw)
def custom_init(self): self._producer = KafkaProducer( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) try: admin_client = KafkaAdminClient( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) admin_client.create_topics([NewTopic(self._queue_name, 16, 1)]) # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)}) except TopicAlreadyExistsError: pass except Exception as e: self.logger.exception(e) atexit.register(self.close) # 程序退出前不主动关闭,会报错。
def test_kafka_lot_of_partitions_partial_commit_of_bulk(kafka_cluster): admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") topic_list = [] topic_list.append( NewTopic(name="topic_with_multiple_partitions2", num_partitions=10, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False) instance.query(''' DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', kafka_topic_list = 'topic_with_multiple_partitions2', kafka_group_name = 'topic_with_multiple_partitions2', kafka_format = 'JSONEachRow', kafka_max_block_size = 211; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree() ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; ''') messages = [] count = 0 for dummy_msg in range(1000): rows = [] for dummy_row in range(random.randrange(3, 10)): count = count + 1 rows.append(json.dumps({'key': count, 'value': count})) messages.append("\n".join(rows)) kafka_produce('topic_with_multiple_partitions2', messages) time.sleep(30) result = instance.query( 'SELECT count(), uniqExact(key), max(key) FROM test.view') print(result) assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(count)) instance.query(''' DROP TABLE test.consumer; DROP TABLE test.view; ''')
class KafkaPublisher( AbstractPublisher, ): """ 使用kafka作为中间件 """ # noinspection PyAttributeOutsideInit def custom_init(self): self._producer = KafkaProducer( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) self._admin_client = KafkaAdminClient( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) try: self._admin_client.create_topics( [NewTopic(self._queue_name, 10, 1)]) # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)}) except TopicAlreadyExistsError: pass except Exception as e: self.logger.exception(e) atexit.register(self.close) # 程序退出前不主动关闭,会报错。 def concrete_realization_of_publish(self, msg): # noinspection PyTypeChecker # self.logger.debug(msg) # print(msg) self._producer.send( self._queue_name, msg.encode(), ) def clear(self): self.logger.warning('还没开始实现 kafka 清空 消息') # self._consumer.seek_to_end() # self.logger.warning(f'将kafka offset 重置到最后位置') def get_message_count(self): # return -1 # 还没找到获取所有分区未消费数量的方法。 # print(self._admin_client.list_consumer_group_offsets('frame_group')) # print(self._admin_client.describe_consumer_groups('frame_group')) return -1 def close(self): self._producer.close() def _at_exit(self): self._producer.flush() super()._at_exit()
def process_notification(self): bootstrap_servers = self._yaml_config.get('bootstrap_servers') read_example_topic = self._yaml_config.get( 'read_example_topic') write_example_topic = self._yaml_config.get( 'write_example_topic') admin_client = KafkaAdminClient( bootstrap_servers=bootstrap_servers) topics = admin_client.list_topics() if read_example_topic in topics: process = Popen(args=[ 'kafka-topics.sh', '--bootstrap-server', bootstrap_servers, '--delete', '--topic', read_example_topic, ], shell=False) print('Delete kafka topic {} status: {}'.format( read_example_topic, process.wait())) if write_example_topic in topics: process = Popen(args=[ 'kafka-topics.sh', '--bootstrap-server', bootstrap_servers, '--delete', '--topic', write_example_topic, ], shell=False) print('Delete kafka topic {} status: {}'.format( write_example_topic, process.wait())) # Create inference online read example topic. admin_client.create_topics(new_topics=[ NewTopic(name=read_example_topic, num_partitions=1, replication_factor=1) ]) # Create inference vector write example topic. admin_client.create_topics(new_topics=[ NewTopic(name=write_example_topic, num_partitions=1, replication_factor=1) ]) self.generate_read_example()
def _createTopicIfNeeded(kafka_connect): admin_client = KafkaAdminClient( bootstrap_servers=kafka_connect['uri'], security_protocol="SSL", ssl_cafile=os.path.join(os.getcwd(), kafka_connect['cafile']), ssl_certfile=os.path.join(os.getcwd(), kafka_connect['certfile']), ssl_keyfile=os.path.join(os.getcwd(), kafka_connect['keyfile'])) try: admin_client.create_topics(new_topics=[ NewTopic(name=kafka_connect['topic'], num_partitions=1, replication_factor=1) ], validate_only=False) admin_client.close() except TopicAlreadyExistsError: pass
def _prepare_topic(self): """ If the target topic is not available yet, it is created. This is only required, if autocreate topic is set to off at the Kafka Broker. :return: - """ consumer = KafkaConsumer( bootstrap_servers=self.producer.config.get('bootstrap_servers')) topics = consumer.topics() if self.target_topic not in topics: client = KafkaAdminClient(bootstrap_servers=self.producer.config. get('bootstrap_servers')) topics = [ NewTopic(self.target_topic, num_partitions=self.num_partitions, replication_factor=self.replication_factor) ] client.create_topics(new_topics=topics)
class Admin: def __init__(self, bootstrap_servers): self.admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers) def create_topic(self, topic): try: topic = NewTopic(name=topic, num_partitions=1, replication_factor=1) self.admin.create_topics([topic], timeout_ms=2000) except Exception: pass def delete_topic(self, topic): res = self.admin.delete_topics([topic], 2000) def close(self): self.admin.close()
def create_topic(): while True: try: admin_client = KafkaAdminClient( bootstrap_servers=BOOTSTRAP_SERVERS, client_id='mps_admin') break except: logger.exception( f'Could not connect to Kafka at {BOOTSTRAP_SERVERS}') time.sleep(1) topic_list = [ NewTopic(name=OUTBOUND_TOPIC_NAME, num_partitions=1, replication_factor=1) ] logger.info(f'Topics List {admin_client.list_topics()}') admin_client.create_topics(new_topics=topic_list, validate_only=False) logger.info(f'Created topic: {OUTBOUND_TOPIC_NAME}')
def custom_init(self): from confluent_kafka import Producer as ConfluentProducer self._producer = KafkaProducer( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) try: admin_client = KafkaAdminClient( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) admin_client.create_topics([NewTopic(self._queue_name, 10, 1)]) # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)}) except TopicAlreadyExistsError: pass except Exception as e: self.logger.exception(e) atexit.register(self.close) # 程序退出前不主动关闭,会报错。 self._confluent_producer = ConfluentProducer({ 'bootstrap.servers': ','.join(frame_config.KAFKA_BOOTSTRAP_SERVERS) }) self._recent_produce_time = time.time()
def _ensure_topics(kafka_config): admin = KafkaAdminClient( bootstrap_servers=kafka_config["bootstrap_servers"], security_protocol="SSL", ssl_cafile=kafka_config["ssl_cafile"], ssl_certfile=kafka_config["ssl_certfile"], ssl_keyfile=kafka_config["ssl_keyfile"]) topic = kafka_config["topic"] num_partitions = kafka_config["num_partitions"] replication_factor = kafka_config["replication_factor"] new_topic = NewTopic(topic, num_partitions, replication_factor) try: logger.info("Ensuring topics %s", topic) admin.create_topics([new_topic]) logger.info("Topic %s created", topic) except TopicAlreadyExistsError: logger.info("Topic %s already exists", topic) return True
def initialize_kafka_topics(self): """ Make sure the server connection is possible and the tracked topics exist """ admin_client = KafkaAdminClient(bootstrap_servers=self._servers, client_id=self._client_id) existing_topics = admin_client.list_topics() if self._request_topic not in existing_topics: request_topic = NewTopic(name=self._request_topic, num_partitions=1, replication_factor=1) admin_client.create_topics(new_topics=[request_topic], validate_only=False) if self._response_topic not in existing_topics: response_topic = NewTopic(name=self._response_topic, num_partitions=1, replication_factor=1) admin_client.create_topics(new_topics=[response_topic], validate_only=False)
def test_create_topics(self): """ Test if topics are correctly created. """ # create topics ... topic_names = ['test_1', 'test2', 'test_3'] # submit the request client = KafkaAdminClient(bootstrap_servers=bootstrap_servers) client.create_topics([ NewTopic(name, num_partitions=1, replication_factor=1) for name in topic_names ]) # request all registered topics consumer = KafkaConsumer(bootstrap_servers=bootstrap_servers) topics = consumer.topics() # check if the requested topics were created for topic in topic_names: assert (topic in topics)
def _shedual_task(self): try: admin_client = KafkaAdminClient( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) admin_client.create_topics([NewTopic(self._queue_name, 10, 1)]) # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)}) except TopicAlreadyExistsError: pass self._producer = KafkaProducer( bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) consumer = OfficialKafkaConsumer( self._queue_name, bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS, group_id=f'frame_group', enable_auto_commit=True, auto_offset_reset='earliest') # auto_offset_reset (str): A policy for resetting offsets on # OffsetOutOfRange errors: 'earliest' will move to the oldest # available message, 'latest' will move to the most recent. Any # other value will raise the exception. Default: 'latest'. 默认是latest # kafka 的 group_id # REMIND 由于是很高数量的并发消费,线程很多,分区很少,这里设置成自动确认消费了,否则多线程提交同一个分区的偏移量导致超前错乱,就没有意义了。 # REMIND 要保证很高的可靠性和一致性,请用rabbitmq。 # REMIND 好处是并发高。topic像翻书一样,随时可以设置偏移量重新消费。多个分组消费同一个主题,每个分组对相同主题的偏移量互不干扰。 for message in consumer: # 注意: message ,value都是原始的字节数据,需要decode if self._is_show_message_get_from_broker: self.logger.debug( f'从kafka的 [{message.topic}] 主题,分区 {message.partition} 中 取出的消息是: {message.value.decode()}' ) kw = { 'consumer': consumer, 'message': message, 'body': json.loads(message.value) } self._submit_task(kw)