def create_topic(self,
topic_name: str,
num_partitions: int = 1,
replication_factor: int = 1):
"""
Creates a new topic if not already present
:param topic_name: New topic name
:param num_partitions: Number of partitions. Default value is 1.
:param replication_factor: Replication factor. Default value is 1.
"""
logger.debug(f"Creating a topic called {topic_name}")
admin = KafkaAdminClient(**self.settings)
existing_topics = admin.list_topics()
if topic_name in existing_topics:
admin.close()
return
topic = [
NewTopic(topic_name,
num_partitions=num_partitions,
replication_factor=replication_factor)
]
admin.create_topics(topic)
admin.close()
logger.debug(f"Topic {topic_name} created")
def run_kafka_server():
# TODO get the json file path
input_file = "police-department-calls-for-service.json" # same folder as this script
# TODO fill in blanks
# GK Note: I have completed the code template provided by Udacity.
# The provided code implies that the topic is autogenerated by KafkaProducer
# The provided code doesn't call the Kafka AdminClient to create the topic explicitly
# To experiment with number of Kafka partitions for performance, I create an admin client
bootstrap_servers = 'localhost:9092'
my_topic = "sf.police.calls"
admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
try:
print("attempting to create new topic")
topic_list = []
nb_partitions = 2
topic_list.append(
NewTopic(name=my_topic,
num_partitions=nb_partitions,
replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
except Exception:
print("topic already exists")
producer = producer_server.ProducerServer(
#producer = producer_server.ProducerServer(
input_file=input_file, # crime events file
topic=my_topic, # freely chosen
bootstrap_servers=bootstrap_servers, # standard choice for Kafka server
client_id="producer-01" # ID chosen for this producer
)
return producer
def create_topic(brokers, topic, partition_count=1, replica_count=1):
"""Create a topic if it does not exist.
Args:
brokers (list): The 'host[:port]' list that the producer should
contact to bootstrap initial cluster metadata.
topic (str): Topic where the message will be published.
partition_count (int): Specified partition number (default 1).
replica_count (int): Specified replication factor (default 1).
Returns:
partitions (set): A set including partition number.
"""
consumer = KafkaConsumer(bootstrap_servers=brokers)
topics = consumer.topics()
if topic in topics:
partitions = consumer.partitions_for_topic(topic)
consumer.close()
else:
consumer.close()
admin = KafkaAdminClient(bootstrap_servers=brokers)
admin.create_topics([
NewTopic(
name=topic,
num_partitions=partition_count,
replication_factor=replica_count,
),
])
admin.close()
partitions = set([p for p in range(partition_count)])
return partitions
def create_topic():
global topic
# Create the kafka topic
tries = 30
exit = False
while not exit:
try:
try:
topic = admin.NewTopic(name="slice",
num_partitions=1,
replication_factor=1)
broker = KafkaAdminClient(bootstrap_servers="kafka:19092")
broker.create_topics([topic])
except errors.TopicAlreadyExistsError:
logger.warning("Topic exists already")
else:
logger.info("New topic")
except errors.NoBrokersAvailable as KafkaError:
if tries > 0:
tries -= 1
logger.warning(
"Kafka not ready yet. Tries remaining: {0}".format(tries))
time.sleep(5)
else:
logger.error(KafkaError)
else:
exit = True
tries = 30
def main():
# Create 'my-topic' Kafka topic
try:
admin = KafkaAdminClient(bootstrap_servers="localhost:9092")
topic = NewTopic(name="my-topic",
num_partitions=1,
replication_factor=1)
admin.create_topics([topic])
except Exception:
pass
tasks = [Producer(), Consumer()]
# Start threads of a publisher/producer and a subscriber/consumer to 'my-topic' Kafka topic
for t in tasks:
t.start()
time.sleep(10)
# Stop threads
for task in tasks:
task.stop()
for task in tasks:
task.join()
def _init_topic(self):
client = KafkaAdminClient(bootstrap_servers=self.servers)
try:
client.create_topics(
[NewTopic(self.topic, num_partitions=len(self.databases), replication_factor=1)]
)
except kafka.errors.TopicAlreadyExistsError:
pass
def main(event, context):
print("Looking up broker ARN")
svc_client = boto3.client('servicediscovery')
response = svc_client.discover_instances(NamespaceName=NAMESPACE,
ServiceName=SERVICE)
broker_arn = response['Instances'][0]['Attributes']['broker_arn']
print("Got broker ARN {0}".format(broker_arn))
print("Looking up broker string")
msk_client = boto3.client('kafka')
response = msk_client.get_bootstrap_brokers(ClusterArn=broker_arn)
broker_string = response['BootstrapBrokerStringTls']
print("Got broker string {0}".format(broker_string))
# make sure topic exists
print("Checking if topic {0} exists".format(TOPIC_NAME))
kclient = KafkaConsumer(bootstrap_servers=broker_string,
security_protocol='SSL')
existing_topics = kclient.topics()
if TOPIC_NAME in existing_topics:
print("Topic {0} exists".format(TOPIC_NAME))
else:
print("Topic {0} does not exist, creating".format(TOPIC_NAME))
topic_list = []
topic = NewTopic(name=TOPIC_NAME,
num_partitions=1,
replication_factor=1)
topic_list.append(topic)
kadmin = KafkaAdminClient(bootstrap_servers=broker_string,
security_protocol='SSL')
kadmin.create_topics(new_topics=topic_list)
kadmin.close()
kclient.close()
producer = KafkaProducer(bootstrap_servers=broker_string,
security_protocol='SSL')
while True:
remaining_time_millis = context.get_remaining_time_in_millis()
if remaining_time_millis < MIN_TIME_REMAINING_MILLIS:
print("Time left ({0}) is less than time required ({1}), exiting".
format(str(remaining_time_millis),
str(MIN_TIME_REMAINING_MILLIS)))
break
else:
print(
"Time left ({0}) is greater than time required ({1}), sending".
format(str(remaining_time_millis),
str(MIN_TIME_REMAINING_MILLIS)))
msg = "Kafka message sent at {0}".format(str(time.time()))
producer.send(TOPIC_NAME, msg.encode('utf-8'))
producer.flush()
time.sleep(SLEEP_SECONDS)
producer.close()
print("All done")
def test_kafka_virtual_columns2(kafka_cluster):
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092")
topic_list = []
topic_list.append(NewTopic(name="virt2_0", num_partitions=2, replication_factor=1))
topic_list.append(NewTopic(name="virt2_1", num_partitions=2, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
instance.query('''
CREATE TABLE test.kafka (value UInt64)
ENGINE = Kafka
SETTINGS kafka_broker_list = 'kafka1:19092',
kafka_topic_list = 'virt2_0,virt2_1',
kafka_group_name = 'virt2',
kafka_format = 'JSONEachRow';
CREATE MATERIALIZED VIEW test.view Engine=Log AS
SELECT value, _key, _topic, _partition, _offset, toUnixTimestamp(_timestamp) FROM test.kafka;
''')
producer = KafkaProducer(bootstrap_servers="localhost:9092")
producer.send(topic='virt2_0', value=json.dumps({'value': 1}), partition=0, key='k1', timestamp_ms=1577836801000)
producer.send(topic='virt2_0', value=json.dumps({'value': 2}), partition=0, key='k2', timestamp_ms=1577836802000)
producer.flush()
time.sleep(1)
producer.send(topic='virt2_0', value=json.dumps({'value': 3}), partition=1, key='k3', timestamp_ms=1577836803000)
producer.send(topic='virt2_0', value=json.dumps({'value': 4}), partition=1, key='k4', timestamp_ms=1577836804000)
producer.flush()
time.sleep(1)
producer.send(topic='virt2_1', value=json.dumps({'value': 5}), partition=0, key='k5', timestamp_ms=1577836805000)
producer.send(topic='virt2_1', value=json.dumps({'value': 6}), partition=0, key='k6', timestamp_ms=1577836806000)
producer.flush()
time.sleep(1)
producer.send(topic='virt2_1', value=json.dumps({'value': 7}), partition=1, key='k7', timestamp_ms=1577836807000)
producer.send(topic='virt2_1', value=json.dumps({'value': 8}), partition=1, key='k8', timestamp_ms=1577836808000)
producer.flush()
time.sleep(10)
result = instance.query("SELECT * FROM test.view ORDER BY value", ignore_error=True)
expected = '''\
1 k1 virt2_0 0 0 1577836801
2 k2 virt2_0 0 1 1577836802
3 k3 virt2_0 1 0 1577836803
4 k4 virt2_0 1 1 1577836804
5 k5 virt2_1 0 0 1577836805
6 k6 virt2_1 0 1 1577836806
7 k7 virt2_1 1 0 1577836807
8 k8 virt2_1 1 1 1577836808
'''
assert TSV(result) == TSV(expected)
def create(topicname):
from kafka import KafkaAdminClient
from kafka.admin import NewTopic
kac = KafkaAdminClient(bootstrap_servers='localhost:9092')
kac.create_topics(new_topics=[
NewTopic(name=topicname, num_partitions=1, replication_factor=1)
])
info_message = 'Created ' + topicname
print(info_message)
return info_message
class Admin:
def __init__(self):
self.client = KafkaAdminClient(bootstrap_servers=app.config['KAFKA_URL'])
def create_topics(self, topics):
topics = [NewTopic(topic, 1, 1) for topic in topics]
self.client.create_topics(new_topics=topics)
def delete_topics(self, topics):
self.client.delete_topics(topics=topics)
def add_topic(args):
admin = KafkaAdminClient(bootstrap_servers=[args.broker])
admin.create_topics([
NewTopic(
name=args.topic,
num_partitions=args.partitions,
replication_factor=args.replication_factor,
),
])
admin.close()
def test_connectivity():
admin = KafkaAdminClient(bootstrap_servers=KAFKA_BROKERS,
security_protocol='SSL')
try:
admin.create_topics([NewTopic(TOPIC_NAME, 3, 2)])
except TopicAlreadyExistsError:
print(f"Topic '{TOPIC_NAME}' already exists.")
producer = KafkaProducer(bootstrap_servers=KAFKA_BROKERS,
security_protocol='SSL')
producer.send(topic=TOPIC_NAME, value=b'yes')
class AdminClient(KafkaPython):
def __init__(self, bootstrap_servers=None, **kwargs):
super().__init__(servers=bootstrap_servers)
admin_client_config.update(kwargs)
self.engine = KafkaAdminClient(
bootstrap_servers=self._bootstrap_servers,
client_id=self._client_id,
request_timeout_ms=self._request_timeout_ms,
**admin_client_config)
def create_topics(self, topic_list: list):
new_topic = []
for k, v in enumerate(topic_list):
new_topic.append(
NewTopics(name=v['name'],
num_partitions=v['num_partitions'],
replication_factor=v['replication_factor'],
replica_assignments=v['replica_assignments'],
topic_configs=v['topic_configs']))
if not Consumer().get_user_topics().intersection(
{item['name']
for i, item in enumerate(topic_list)}):
try:
self.engine.create_topics(new_topic, **create_topic_config)
except KafkaError as e:
_logger.error(e)
self.engine.close()
else:
_logger.error(
self._logMsg(create_topic_fail_code, self._client_id,
'topic重复'))
return
def delete_topics(self, topic: list):
if Consumer().get_user_topics().intersection(set(topic)):
try:
self.engine.delete_topics(topic, self._request_timeout_ms)
except KafkaError as e:
_logger.error(
self._logMsg(delete_topic_fail_code, self._client_id,
'删除的topic失败:%s' % e))
self.engine.close()
else:
_logger.error(
self._logMsg(delete_topic_fail_code, self._client_id,
'需删除的topic不存在'))
return
def createTopic(self, bootstrapServer, topic, numPartitions,
replicationFactor):
try:
admin = KafkaAdminClient(bootstrapServer)
topic = NewTopic(name=topic,
num_partitions=numPartitions,
replication_factor=replicationFactor)
admin.create_topics([topic])
except Exception as e:
logger.error('Error to create topic %s' % e)
pass
def ensure_kafka_topic_exists(config):
set_up = False
for i in range(1, 12):
try:
admin_client = KafkaAdminClient(bootstrap_servers=config.brokers)
admin_client.create_topics([NewTopic(config.topic, 1, 1)])
set_up = True
break
except (UnrecognizedBrokerVersion, ValueError):
time.sleep(5)
if not set_up:
raise Exception('Failed it setup kafka topic')
def main():
# Create 'my-topic' Kafka topic
try:
admin = KafkaAdminClient(bootstrap_servers='localhost:9092')
topic = NewTopic(name='outputframes',
num_partitions=1,
replication_factor=1)
admin.create_topics([topic])
except Exception:
pass
subscribe()
def _shedual_task(self):
from confluent_kafka import Consumer as ConfluentConsumer # 这个包不好安装，用户用这个中间件的时候自己再想办法安装。
try:
admin_client = KafkaAdminClient(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
# admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
except TopicAlreadyExistsError:
pass
self._producer = KafkaProducer(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
# consumer 配置 https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
self._confluent_consumer = ConfluentConsumer({
'bootstrap.servers':
','.join(frame_config.KAFKA_BOOTSTRAP_SERVERS),
'group.id':
'frame_group',
'auto.offset.reset':
'earliest',
'enable.auto.commit':
False
})
self._confluent_consumer.subscribe([self._queue_name])
self._recent_commit_time = time.time()
self._partion__offset_consume_status_map = defaultdict(OrderedDict)
while 1:
msg = self._confluent_consumer.poll(timeout=10)
self._manually_commit()
if msg is None:
continue
if msg.error():
print("Consumer error: {}".format(msg.error()))
continue
# msg的类型 https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#message
# value() offset() partition()
# print('Received message: {}'.format(msg.value().decode('utf-8'))) # noqa
self._partion__offset_consume_status_map[msg.partition()][
msg.offset()] = 0
kw = {
'partition': msg.partition(),
'offset': msg.offset(),
'body': json.loads(msg.value())
} # noqa
if self._is_show_message_get_from_broker:
self.logger.debug(
f'从kafka的 [{self._queue_name}] 主题,分区 {msg.partition()} 中 的 offset {msg.offset()} 取出的消息是： {msg.value()}'
) # noqa
self._submit_task(kw)
def custom_init(self):
self._producer = KafkaProducer(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
try:
admin_client = KafkaAdminClient(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
admin_client.create_topics([NewTopic(self._queue_name, 16, 1)])
# admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
except TopicAlreadyExistsError:
pass
except Exception as e:
self.logger.exception(e)
atexit.register(self.close) # 程序退出前不主动关闭，会报错。
def test_kafka_lot_of_partitions_partial_commit_of_bulk(kafka_cluster):
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092")
topic_list = []
topic_list.append(
NewTopic(name="topic_with_multiple_partitions2",
num_partitions=10,
replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
instance.query('''
DROP TABLE IF EXISTS test.view;
DROP TABLE IF EXISTS test.consumer;
CREATE TABLE test.kafka (key UInt64, value UInt64)
ENGINE = Kafka
SETTINGS kafka_broker_list = 'kafka1:19092',
kafka_topic_list = 'topic_with_multiple_partitions2',
kafka_group_name = 'topic_with_multiple_partitions2',
kafka_format = 'JSONEachRow',
kafka_max_block_size = 211;
CREATE TABLE test.view (key UInt64, value UInt64)
ENGINE = MergeTree()
ORDER BY key;
CREATE MATERIALIZED VIEW test.consumer TO test.view AS
SELECT * FROM test.kafka;
''')
messages = []
count = 0
for dummy_msg in range(1000):
rows = []
for dummy_row in range(random.randrange(3, 10)):
count = count + 1
rows.append(json.dumps({'key': count, 'value': count}))
messages.append("\n".join(rows))
kafka_produce('topic_with_multiple_partitions2', messages)
time.sleep(30)
result = instance.query(
'SELECT count(), uniqExact(key), max(key) FROM test.view')
print(result)
assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(count))
instance.query('''
DROP TABLE test.consumer;
DROP TABLE test.view;
''')
class KafkaPublisher(
AbstractPublisher, ):
"""
使用kafka作为中间件
"""
# noinspection PyAttributeOutsideInit
def custom_init(self):
self._producer = KafkaProducer(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
self._admin_client = KafkaAdminClient(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
try:
self._admin_client.create_topics(
[NewTopic(self._queue_name, 10, 1)])
# admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
except TopicAlreadyExistsError:
pass
except Exception as e:
self.logger.exception(e)
atexit.register(self.close) # 程序退出前不主动关闭，会报错。
def concrete_realization_of_publish(self, msg):
# noinspection PyTypeChecker
# self.logger.debug(msg)
# print(msg)
self._producer.send(
self._queue_name,
msg.encode(),
)
def clear(self):
self.logger.warning('还没开始实现 kafka 清空 消息')
# self._consumer.seek_to_end()
# self.logger.warning(f'将kafka offset 重置到最后位置')
def get_message_count(self):
# return -1 # 还没找到获取所有分区未消费数量的方法。
# print(self._admin_client.list_consumer_group_offsets('frame_group'))
# print(self._admin_client.describe_consumer_groups('frame_group'))
return -1
def close(self):
self._producer.close()
def _at_exit(self):
self._producer.flush()
super()._at_exit()
def process_notification(self):
bootstrap_servers = self._yaml_config.get('bootstrap_servers')
read_example_topic = self._yaml_config.get(
'read_example_topic')
write_example_topic = self._yaml_config.get(
'write_example_topic')
admin_client = KafkaAdminClient(
bootstrap_servers=bootstrap_servers)
topics = admin_client.list_topics()
if read_example_topic in topics:
process = Popen(args=[
'kafka-topics.sh',
'--bootstrap-server',
bootstrap_servers,
'--delete',
'--topic',
read_example_topic,
],
shell=False)
print('Delete kafka topic {} status: {}'.format(
read_example_topic, process.wait()))
if write_example_topic in topics:
process = Popen(args=[
'kafka-topics.sh',
'--bootstrap-server',
bootstrap_servers,
'--delete',
'--topic',
write_example_topic,
],
shell=False)
print('Delete kafka topic {} status: {}'.format(
write_example_topic, process.wait()))
# Create inference online read example topic.
admin_client.create_topics(new_topics=[
NewTopic(name=read_example_topic,
num_partitions=1,
replication_factor=1)
])
# Create inference vector write example topic.
admin_client.create_topics(new_topics=[
NewTopic(name=write_example_topic,
num_partitions=1,
replication_factor=1)
])
self.generate_read_example()
def _createTopicIfNeeded(kafka_connect):
admin_client = KafkaAdminClient(
bootstrap_servers=kafka_connect['uri'],
security_protocol="SSL",
ssl_cafile=os.path.join(os.getcwd(), kafka_connect['cafile']),
ssl_certfile=os.path.join(os.getcwd(), kafka_connect['certfile']),
ssl_keyfile=os.path.join(os.getcwd(), kafka_connect['keyfile']))
try:
admin_client.create_topics(new_topics=[
NewTopic(name=kafka_connect['topic'],
num_partitions=1,
replication_factor=1)
],
validate_only=False)
admin_client.close()
except TopicAlreadyExistsError:
pass
def _prepare_topic(self):
"""
If the target topic is not available yet, it is created. This is only required, if autocreate topic is set
to off at the Kafka Broker.
:return: -
"""
consumer = KafkaConsumer(
bootstrap_servers=self.producer.config.get('bootstrap_servers'))
topics = consumer.topics()
if self.target_topic not in topics:
client = KafkaAdminClient(bootstrap_servers=self.producer.config.
get('bootstrap_servers'))
topics = [
NewTopic(self.target_topic,
num_partitions=self.num_partitions,
replication_factor=self.replication_factor)
]
client.create_topics(new_topics=topics)
class Admin:
def __init__(self, bootstrap_servers):
self.admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
def create_topic(self, topic):
try:
topic = NewTopic(name=topic,
num_partitions=1,
replication_factor=1)
self.admin.create_topics([topic], timeout_ms=2000)
except Exception:
pass
def delete_topic(self, topic):
res = self.admin.delete_topics([topic], 2000)
def close(self):
self.admin.close()
def create_topic():
while True:
try:
admin_client = KafkaAdminClient(
bootstrap_servers=BOOTSTRAP_SERVERS, client_id='mps_admin')
break
except:
logger.exception(
f'Could not connect to Kafka at {BOOTSTRAP_SERVERS}')
time.sleep(1)
topic_list = [
NewTopic(name=OUTBOUND_TOPIC_NAME,
num_partitions=1,
replication_factor=1)
]
logger.info(f'Topics List {admin_client.list_topics()}')
admin_client.create_topics(new_topics=topic_list, validate_only=False)
logger.info(f'Created topic: {OUTBOUND_TOPIC_NAME}')
def custom_init(self):
from confluent_kafka import Producer as ConfluentProducer
self._producer = KafkaProducer(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
try:
admin_client = KafkaAdminClient(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
# admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
except TopicAlreadyExistsError:
pass
except Exception as e:
self.logger.exception(e)
atexit.register(self.close) # 程序退出前不主动关闭，会报错。
self._confluent_producer = ConfluentProducer({
'bootstrap.servers':
','.join(frame_config.KAFKA_BOOTSTRAP_SERVERS)
})
self._recent_produce_time = time.time()
def _ensure_topics(kafka_config):
admin = KafkaAdminClient(
bootstrap_servers=kafka_config["bootstrap_servers"],
security_protocol="SSL",
ssl_cafile=kafka_config["ssl_cafile"],
ssl_certfile=kafka_config["ssl_certfile"],
ssl_keyfile=kafka_config["ssl_keyfile"])
topic = kafka_config["topic"]
num_partitions = kafka_config["num_partitions"]
replication_factor = kafka_config["replication_factor"]
new_topic = NewTopic(topic, num_partitions, replication_factor)
try:
logger.info("Ensuring topics %s", topic)
admin.create_topics([new_topic])
logger.info("Topic %s created", topic)
except TopicAlreadyExistsError:
logger.info("Topic %s already exists", topic)
return True
def initialize_kafka_topics(self):
"""
Make sure the server connection is possible and the tracked topics exist
"""
admin_client = KafkaAdminClient(bootstrap_servers=self._servers,
client_id=self._client_id)
existing_topics = admin_client.list_topics()
if self._request_topic not in existing_topics:
request_topic = NewTopic(name=self._request_topic,
num_partitions=1,
replication_factor=1)
admin_client.create_topics(new_topics=[request_topic],
validate_only=False)
if self._response_topic not in existing_topics:
response_topic = NewTopic(name=self._response_topic,
num_partitions=1,
replication_factor=1)
admin_client.create_topics(new_topics=[response_topic],
validate_only=False)
def test_create_topics(self):
"""
Test if topics are correctly created.
"""
# create topics ...
topic_names = ['test_1', 'test2', 'test_3']
# submit the request
client = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
client.create_topics([
NewTopic(name, num_partitions=1, replication_factor=1)
for name in topic_names
])
# request all registered topics
consumer = KafkaConsumer(bootstrap_servers=bootstrap_servers)
topics = consumer.topics()
# check if the requested topics were created
for topic in topic_names:
assert (topic in topics)
def _shedual_task(self):
try:
admin_client = KafkaAdminClient(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
# admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
except TopicAlreadyExistsError:
pass
self._producer = KafkaProducer(
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS)
consumer = OfficialKafkaConsumer(
self._queue_name,
bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS,
group_id=f'frame_group',
enable_auto_commit=True,
auto_offset_reset='earliest')
# auto_offset_reset (str): A policy for resetting offsets on
# OffsetOutOfRange errors: 'earliest' will move to the oldest
# available message, 'latest' will move to the most recent. Any
# other value will raise the exception. Default: 'latest'. 默认是latest
# kafka 的 group_id
# REMIND 由于是很高数量的并发消费，线程很多，分区很少，这里设置成自动确认消费了，否则多线程提交同一个分区的偏移量导致超前错乱，就没有意义了。
# REMIND 要保证很高的可靠性和一致性，请用rabbitmq。
# REMIND 好处是并发高。topic像翻书一样，随时可以设置偏移量重新消费。多个分组消费同一个主题，每个分组对相同主题的偏移量互不干扰。
for message in consumer:
# 注意: message ,value都是原始的字节数据，需要decode
if self._is_show_message_get_from_broker:
self.logger.debug(
f'从kafka的 [{message.topic}] 主题,分区 {message.partition} 中 取出的消息是： {message.value.decode()}'
)
kw = {
'consumer': consumer,
'message': message,
'body': json.loads(message.value)
}
self._submit_task(kw)
