def backup(self): # Read topic messages from kafka broker if self.config.kafka_library == 'confluent': reader = ConfluentKafkaReader(self.config.brokers) else: reader = PythonKafkaReader(self.config.brokers) if self.config.ignore_missing_topics: logger.debug('Filter out topics that are not in Kafka broker') broker_topic_names = reader.list_topics() topics = [] for t in self.config.topics: if t not in broker_topic_names: logger.debug(f'Ignore topic {t} since it is ' 'missing in kafka broker') continue topics.append(t) else: topics = self.config.topics reader.subscribe(topics) msg_dict = reader.read(timeout=self.config.consumer_timeout) partitions = [ Partition(topic, partition_no, msgs) for (topic, partition_no), msgs in msg_dict.items() ] # Fetch consumer group offsets admin_client = ConfluentAdminClient(self.config.brokers) offsets = admin_client.get_consumer_offsets( topics, no_of_threads=self.config.threads) # Write topic messages and consumer offsets to disk data_flow_manager = DataFlowManager(self.config.data) data_flow_manager.write(offsets, partitions)
def test_confluent_admin_client_get_consumer_offsets(monkeypatch): monkeypatch.setattr(confluent_kafka, 'Consumer', MockedConsumer) confluent_admin_client = ConfluentAdminClient(['foo_baz_1']) confluent_admin_client.client = MockedAdmin() offsets = confluent_admin_client.get_consumer_offsets(['Topic1']) assert offsets offsets = confluent_admin_client.get_consumer_offsets(['Topic1'], no_of_threads=2) assert offsets
def restore(self): # Read topic messages and consumer offsets from disk data_flow_manager = DataFlowManager(self.config.data) offsets, partitions = data_flow_manager.read(self.config.topics) def func(partition): logger.debug(f'Write {len(partition.messages)} messages' f'to topic: {partition.topic} ' f'partition: {partition.name} in kafka broker') writer = ConfluentKafkaWriter(self.config.brokers) for msg in partition.messages: writer.write(partition.topic, partition.name, msg) writer.flush() # Write topic messages to kafka broker pool = ThreadPool(self.config.threads) pool.map(func, partitions) # Calculate new offsets new_offsets = generate_new_offsets(offsets, partitions) # Set consumer group offsets admin_client = ConfluentAdminClient(self.config.brokers) admin_client.set_consumer_offsets(new_offsets)
def test_confluent_kafka_writer_write( confluent_kafka_writer: ConfluentKafkaWriter): confluent_admin_client = ConfluentAdminClient(['localhost']) new_topic = NewTopic('Topic1', num_partitions=1, replication_factor=1) try: future_dict = confluent_admin_client.client.create_topics([new_topic]) wait_for_futures(future_dict) except: pass confluent_kafka_writer.write( 'Topic1', 0, Message(1, 'Key1', 'Val1', int(datetime.now().timestamp()))) sleep(5) confluent_kafka_reader = ConfluentKafkaReader(['localhost']) confluent_kafka_reader.subscribe(['Topic1']) partition_message_dict = confluent_kafka_reader.read() assert partition_message_dict
def test_get_consumer_groups(confluent_admin_client: ConfluentAdminClient): consumer_groups = confluent_admin_client.get_consumer_groups() assert consumer_groups
def confluent_admin_client(KAFKA_HOSTS): return ConfluentAdminClient(KAFKA_HOSTS)
def test_get_consumer_offsets(confluent_admin_client: ConfluentAdminClient): offsets = confluent_admin_client.get_consumer_offsets( ['CBBCAutomationInstrumentFitPOParameters'], no_of_threads=8) assert offsets