def test_send_offsets_committed_transaction(kafka_cluster): input_topic = kafka_cluster.create_topic("input_topic") output_topic = kafka_cluster.create_topic("output_topic") error_cb = prefixed_error_cb('test_send_offsets_committed_transaction') producer = kafka_cluster.producer({ 'client.id': 'producer1', 'transactional.id': 'example_transactional_id', 'error_cb': error_cb, }) consumer_conf = { 'group.id': str(uuid1()), 'auto.offset.reset': 'earliest', 'enable.auto.commit': False, 'enable.partition.eof': True, 'error_cb': error_cb } consumer_conf.update(kafka_cluster.client_conf()) consumer = Consumer(consumer_conf) kafka_cluster.seed_topic(input_topic) consumer.subscribe([input_topic]) read_all_msgs(consumer) producer.init_transactions() transactional_produce(producer, output_topic, 100) consumer_position = consumer.position(consumer.assignment()) group_metadata = consumer.consumer_group_metadata() print( "=== Sending offsets {} to transaction ===".format(consumer_position)) producer.send_offsets_to_transaction(consumer_position, group_metadata) producer.commit_transaction() producer2 = kafka_cluster.producer({ 'client.id': 'producer2', 'transactional.id': 'example_transactional_id', 'error_cb': error_cb }) # ensure offset commits are visible prior to sending FetchOffsets request producer2.init_transactions() committed_offsets = consumer.committed(consumer.assignment()) print("=== Committed offsets for {} ===".format(committed_offsets)) assert [tp.offset for tp in committed_offsets] == [100] consumer.close()
def test_transaction_api(): """ Excercise the transactional API """ p = Producer({"transactional.id": "test"}) with pytest.raises(KafkaException) as ex: p.init_transactions(0.5) assert ex.value.args[0].code() == KafkaError._TIMED_OUT assert ex.value.args[0].retriable() is True assert ex.value.args[0].fatal() is False assert ex.value.args[0].txn_requires_abort() is False # Any subsequent APIs will fail since init did not succeed. with pytest.raises(KafkaException) as ex: p.begin_transaction() assert ex.value.args[0].code() == KafkaError._STATE assert ex.value.args[0].retriable() is False assert ex.value.args[0].fatal() is False assert ex.value.args[0].txn_requires_abort() is False consumer = Consumer({"group.id": "testgroup"}) group_metadata = consumer.consumer_group_metadata() consumer.close() with pytest.raises(KafkaException) as ex: p.send_offsets_to_transaction([TopicPartition("topic", 0, 123)], group_metadata) assert ex.value.args[0].code() == KafkaError._STATE assert ex.value.args[0].retriable() is False assert ex.value.args[0].fatal() is False assert ex.value.args[0].txn_requires_abort() is False with pytest.raises(KafkaException) as ex: p.commit_transaction(0.5) assert ex.value.args[0].code() == KafkaError._STATE assert ex.value.args[0].retriable() is False assert ex.value.args[0].fatal() is False assert ex.value.args[0].txn_requires_abort() is False with pytest.raises(KafkaException) as ex: p.abort_transaction(0.5) assert ex.value.args[0].code() == KafkaError._STATE assert ex.value.args[0].retriable() is False assert ex.value.args[0].fatal() is False assert ex.value.args[0].txn_requires_abort() is False
def main(args): brokers = args.brokers group_id = args.group_id input_topic = args.input_topic input_partition = args.input_partition output_topic = args.output_topic consumer = Consumer({ 'bootstrap.servers': brokers, 'group.id': group_id, 'auto.offset.reset': 'earliest', # Do not advance committed offsets outside of the transaction. # Consumer offsets are committed along with the transaction # using the producer's send_offsets_to_transaction() API. 'enable.auto.commit': False, 'enable.partition.eof': True, }) # Prior to KIP-447 being supported each input partition requires # its own transactional producer, so in this example we use # assign() to a single partition rather than subscribe(). # A more complex alternative is to dynamically create a producer per # partition in subscribe's rebalance callback. consumer.assign([TopicPartition(input_topic, input_partition)]) producer = Producer({ 'bootstrap.servers': brokers, 'transactional.id': 'eos-transactions.py' }) # Initialize producer transaction. producer.init_transactions() # Start producer transaction. producer.begin_transaction() eof = {} msg_cnt = 0 print("=== Starting Consume-Transform-Process loop ===") while True: # serve delivery reports from previous produce()s producer.poll(0) # read message from input_topic msg = consumer.poll(timeout=1.0) if msg is None: continue topic, partition = msg.topic(), msg.partition() if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: eof[(topic, partition)] = True print("=== Reached the end of {} [{}] at {}====".format( topic, partition, msg.offset())) if len(eof) == len(consumer.assignment()): print("=== Reached end of input ===") break continue # clear EOF if a new message has been received eof.pop((topic, partition), None) msg_cnt += 1 # process message processed_key, processed_value = process_input(msg) # produce transformed message to output topic producer.produce(output_topic, processed_value, processed_key, on_delivery=delivery_report) if msg_cnt % 100 == 0: print( "=== Committing transaction with {} messages at input offset {} ===" .format(msg_cnt, msg.offset())) # Send the consumer's position to transaction to commit # them along with the transaction, committing both # input and outputs in the same transaction is what provides EOS. producer.send_offsets_to_transaction( consumer.position(consumer.assignment()), consumer.consumer_group_metadata()) # Commit the transaction producer.commit_transaction() # Begin new transaction producer.begin_transaction() msg_cnt = 0 print("=== Committing final transaction with {} messages ===".format( msg_cnt)) # commit processed message offsets to the transaction producer.send_offsets_to_transaction( consumer.position(consumer.assignment()), consumer.consumer_group_metadata()) # commit transaction producer.commit_transaction() consumer.close()
timestamp=record_json.get("phenomenonTime"), result=record_json.get("result"), topic=msg.topic(), partition=msg.partition(), offset=msg.offset(), **additional_attributes) # ingest the record into the StreamBuffer instance, instant emit if record.get("topic") == KAFKA_TOPIC_IN_1: # Car1 stream_buffer.ingest_left(record) # with instant emit elif record.get("topic") == KAFKA_TOPIC_IN_2: # Car2 stream_buffer.ingest_right(record) except KeyboardInterrupt: print("Gracefully stopping") finally: ts_stop = time.time() # commit processed message offsets to the transaction kafka_producer.send_offsets_to_transaction( kafka_consumer.position(kafka_consumer.assignment()), kafka_consumer.consumer_group_metadata()) # commit transaction kafka_producer.commit_transaction() # Leave group and commit offsets kafka_consumer.close() print(f"\nRecords in |{KAFKA_TOPIC_OUT}| = {stream_buffer.get_join_counter()}, " f"|{KAFKA_TOPIC_IN_1}| = {stream_buffer.get_left_counter()}, " f"|{KAFKA_TOPIC_IN_2}| = {stream_buffer.get_right_counter()}.") print(f"Joined time-series {ts_stop - st0:.5g} s long, " f"this are {stream_buffer.get_join_counter() / (ts_stop - st0):.6g} joins per second.")