def test_transaction_api():
""" Excercise the transactional API """
p = Producer({"transactional.id": "test"})
with pytest.raises(KafkaException) as ex:
p.init_transactions(0.5)
assert ex.value.args[0].code() == KafkaError._TIMED_OUT
assert ex.value.args[0].retriable() is True
assert ex.value.args[0].fatal() is False
assert ex.value.args[0].txn_requires_abort() is False
# Any subsequent APIs will fail since init did not succeed.
with pytest.raises(KafkaException) as ex:
p.begin_transaction()
assert ex.value.args[0].code() == KafkaError._STATE
assert ex.value.args[0].retriable() is False
assert ex.value.args[0].fatal() is False
assert ex.value.args[0].txn_requires_abort() is False
consumer = Consumer({"group.id": "testgroup"})
group_metadata = consumer.consumer_group_metadata()
consumer.close()
with pytest.raises(KafkaException) as ex:
p.send_offsets_to_transaction([TopicPartition("topic", 0, 123)],
group_metadata)
assert ex.value.args[0].code() == KafkaError._STATE
assert ex.value.args[0].retriable() is False
assert ex.value.args[0].fatal() is False
assert ex.value.args[0].txn_requires_abort() is False
with pytest.raises(KafkaException) as ex:
p.commit_transaction(0.5)
assert ex.value.args[0].code() == KafkaError._STATE
assert ex.value.args[0].retriable() is False
assert ex.value.args[0].fatal() is False
assert ex.value.args[0].txn_requires_abort() is False
with pytest.raises(KafkaException) as ex:
p.abort_transaction(0.5)
assert ex.value.args[0].code() == KafkaError._STATE
assert ex.value.args[0].retriable() is False
assert ex.value.args[0].fatal() is False
assert ex.value.args[0].txn_requires_abort() is False
def main(args):
brokers = args.brokers
group_id = args.group_id
input_topic = args.input_topic
input_partition = args.input_partition
output_topic = args.output_topic
consumer = Consumer({
'bootstrap.servers': brokers,
'group.id': group_id,
'auto.offset.reset': 'earliest',
# Do not advance committed offsets outside of the transaction.
# Consumer offsets are committed along with the transaction
# using the producer's send_offsets_to_transaction() API.
'enable.auto.commit': False,
'enable.partition.eof': True,
})
# Prior to KIP-447 being supported each input partition requires
# its own transactional producer, so in this example we use
# assign() to a single partition rather than subscribe().
# A more complex alternative is to dynamically create a producer per
# partition in subscribe's rebalance callback.
consumer.assign([TopicPartition(input_topic, input_partition)])
producer = Producer({
'bootstrap.servers': brokers,
'transactional.id': 'eos-transactions.py'
})
# Initialize producer transaction.
producer.init_transactions()
# Start producer transaction.
producer.begin_transaction()
eof = {}
msg_cnt = 0
print("=== Starting Consume-Transform-Process loop ===")
while True:
# serve delivery reports from previous produce()s
producer.poll(0)
# read message from input_topic
msg = consumer.poll(timeout=1.0)
if msg is None:
continue
topic, partition = msg.topic(), msg.partition()
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
eof[(topic, partition)] = True
print("=== Reached the end of {} [{}] at {}====".format(
topic, partition, msg.offset()))
if len(eof) == len(consumer.assignment()):
print("=== Reached end of input ===")
break
continue
# clear EOF if a new message has been received
eof.pop((topic, partition), None)
msg_cnt += 1
# process message
processed_key, processed_value = process_input(msg)
# produce transformed message to output topic
producer.produce(output_topic,
processed_value,
processed_key,
on_delivery=delivery_report)
if msg_cnt % 100 == 0:
print(
"=== Committing transaction with {} messages at input offset {} ==="
.format(msg_cnt, msg.offset()))
# Send the consumer's position to transaction to commit
# them along with the transaction, committing both
# input and outputs in the same transaction is what provides EOS.
producer.send_offsets_to_transaction(
consumer.position(consumer.assignment()),
consumer.consumer_group_metadata())
# Commit the transaction
producer.commit_transaction()
# Begin new transaction
producer.begin_transaction()
msg_cnt = 0
print("=== Committing final transaction with {} messages ===".format(
msg_cnt))
# commit processed message offsets to the transaction
producer.send_offsets_to_transaction(
consumer.position(consumer.assignment()),
consumer.consumer_group_metadata())
# commit transaction
producer.commit_transaction()
consumer.close()
timestamp=record_json.get("phenomenonTime"),
result=record_json.get("result"),
topic=msg.topic(), partition=msg.partition(), offset=msg.offset(),
**additional_attributes)
# ingest the record into the StreamBuffer instance, instant emit
if record.get("topic") == KAFKA_TOPIC_IN_1: # Car1
stream_buffer.ingest_left(record) # with instant emit
elif record.get("topic") == KAFKA_TOPIC_IN_2: # Car2
stream_buffer.ingest_right(record)
except KeyboardInterrupt:
print("Gracefully stopping")
finally:
ts_stop = time.time()
# commit processed message offsets to the transaction
kafka_producer.send_offsets_to_transaction(
kafka_consumer.position(kafka_consumer.assignment()),
kafka_consumer.consumer_group_metadata())
# commit transaction
kafka_producer.commit_transaction()
# Leave group and commit offsets
kafka_consumer.close()
print(f"\nRecords in |{KAFKA_TOPIC_OUT}| = {stream_buffer.get_join_counter()}, "
f"|{KAFKA_TOPIC_IN_1}| = {stream_buffer.get_left_counter()}, "
f"|{KAFKA_TOPIC_IN_2}| = {stream_buffer.get_right_counter()}.")
print(f"Joined time-series {ts_stop - st0:.5g} s long, "
f"this are {stream_buffer.get_join_counter() / (ts_stop - st0):.6g} joins per second.")
