def _setup_error_after_data(self):
subscriptions = SubscriptionState('latest')
client = AIOKafkaClient(
loop=self.loop,
bootstrap_servers=[])
fetcher = Fetcher(client, subscriptions, loop=self.loop)
tp1 = TopicPartition('some_topic', 0)
tp2 = TopicPartition('some_topic', 1)
state = TopicPartitionState()
state.seek(0)
subscriptions.assignment[tp1] = state
state = TopicPartitionState()
state.seek(0)
subscriptions.assignment[tp2] = state
subscriptions.needs_partition_assignment = False
# Add some data
messages = [ConsumerRecord(
topic="some_topic", partition=1, offset=0, timestamp=0,
timestamp_type=0, key=None, value=b"some", checksum=None,
serialized_key_size=0, serialized_value_size=4)]
fetcher._records[tp2] = FetchResult(
tp2, subscriptions=subscriptions, loop=self.loop,
records=iter(messages), backoff=0)
# Add some error
fetcher._records[tp1] = FetchError(
loop=self.loop, error=OffsetOutOfRangeError({}), backoff=0)
return fetcher, tp1, tp2, messages
def _setup_error_after_data(self):
subscriptions = SubscriptionState(loop=self.loop)
client = AIOKafkaClient(
loop=self.loop,
bootstrap_servers=[])
fetcher = Fetcher(client, subscriptions, loop=self.loop)
tp1 = TopicPartition('some_topic', 0)
tp2 = TopicPartition('some_topic', 1)
subscriptions.subscribe(set(["some_topic"]))
subscriptions.assign_from_subscribed({tp1, tp2})
assignment = subscriptions.subscription.assignment
subscriptions.seek(tp1, 0)
subscriptions.seek(tp2, 0)
# Add some data
messages = [ConsumerRecord(
topic="some_topic", partition=1, offset=0, timestamp=0,
timestamp_type=0, key=None, value=b"some", checksum=None,
serialized_key_size=0, serialized_value_size=4)]
fetcher._records[tp2] = FetchResult(
tp2, assignment=assignment, loop=self.loop,
message_iterator=iter(messages), backoff=0,
fetch_offset=0)
# Add some error
fetcher._records[tp1] = FetchError(
loop=self.loop, error=OffsetOutOfRangeError({}), backoff=0)
return fetcher, tp1, tp2, messages
async def test_sender__do_txn_offset_commit_ok(self):
sender = await self._setup_sender()
offsets = {
TopicPartition("topic", 0): OffsetAndMetadata(10, ""),
TopicPartition("topic", 1): OffsetAndMetadata(11, ""),
}
add_handler = TxnOffsetCommitHandler(sender, offsets, "some_group")
tm = sender._txn_manager
tm.offset_committed = mock.Mock()
# Handle response
cls = TxnOffsetCommitResponse[0]
resp = cls(
throttle_time_ms=300,
errors=[
("topic", [
(0, NoError.errno),
(1, NoError.errno)
])
]
)
backoff = add_handler.handle_response(resp)
self.assertIsNone(backoff)
self.assertEqual(tm.offset_committed.call_count, 2)
tm.offset_committed.assert_has_calls([
mock.call(TopicPartition("topic", 0), 10, "some_group"),
mock.call(TopicPartition("topic", 1), 11, "some_group"),
])
def test_txn_manager(txn_manager):
assert txn_manager._pid_and_epoch.pid == NO_PRODUCER_ID
assert txn_manager._pid_and_epoch.epoch == NO_PRODUCER_EPOCH
assert not txn_manager._sequence_numbers
assert not txn_manager.has_pid()
txn_manager.set_pid_and_epoch(123, 321)
assert txn_manager._pid_and_epoch.pid == 123
assert txn_manager._pid_and_epoch.epoch == 321
assert not txn_manager._sequence_numbers
assert txn_manager.has_pid()
tp1 = TopicPartition("topic", 1)
tp2 = TopicPartition("topic", 2)
assert txn_manager.sequence_number(tp1) == 0
txn_manager.increment_sequence_number(tp1, 1)
assert txn_manager.sequence_number(tp1) == 1
# Changing one sequence_number does not change other
assert txn_manager.sequence_number(tp2) == 0
txn_manager.increment_sequence_number(tp2, 33)
assert txn_manager.sequence_number(tp2) == 33
assert txn_manager.sequence_number(tp1) == 1
# sequence number should wrap around 32 bit signed integers
txn_manager.increment_sequence_number(tp1, 2**32 - 5)
assert txn_manager.sequence_number(tp1) == -4
async def test_record_metric_on_rebalance():
async def coro(*arg, **kwargs):
pass
with patch("kafkaesk.consumer.CONSUMER_REBALANCED") as rebalance_metric:
app_mock = AsyncMock()
app_mock.topic_mng.list_consumer_group_offsets.return_value = {
TopicPartition(topic="foobar", partition=0):
OffsetAndMetadata(offset=0, metadata={})
}
subscription = Subscription(
"test_consumer",
coro,
"group",
topics=["stream.foo"],
)
rebalance_listener = BatchConsumer(
subscription=subscription,
app=app_mock,
)
rebalance_listener._consumer = AsyncMock()
await rebalance_listener.on_partitions_assigned(
[TopicPartition(topic="foobar", partition=0)])
rebalance_metric.labels.assert_called_with(
partition=0,
group_id="group",
event="assigned",
)
rebalance_metric.labels().inc.assert_called_once()
async def test_sender__do_add_partitions_to_txn_ok(self):
sender = await self._setup_sender()
tps = [
TopicPartition("topic", 0),
TopicPartition("topic", 1),
TopicPartition("topic2", 1)
]
add_handler = AddPartitionsToTxnHandler(sender, tps)
tm = sender._txn_manager
tm.partition_added = mock.Mock()
# Handle response
cls = AddPartitionsToTxnResponse[0]
resp = cls(
throttle_time_ms=300,
errors=[
("topic", [
(0, NoError.errno),
(1, NoError.errno)
]),
("topic2", [
(1, NoError.errno)
])
]
)
backoff = add_handler.handle_response(resp)
self.assertIsNone(backoff)
self.assertEqual(tm.partition_added.call_count, 3)
tm.partition_added.assert_has_calls([
mock.call(tp) for tp in tps
])
async def test_consumer_transactional_commit(self):
producer = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts,
transactional_id="sobaka_producer")
await producer.start()
self.add_cleanup(producer.stop)
producer2 = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts)
await producer2.start()
self.add_cleanup(producer2.stop)
consumer = AIOKafkaConsumer(
self.topic, loop=self.loop,
bootstrap_servers=self.hosts,
auto_offset_reset="earliest",
isolation_level="read_committed")
await consumer.start()
self.add_cleanup(consumer.stop)
# We will produce from a transactional producer and then from a
# non-transactional. This should block consumption on that partition
# until transaction is committed.
await producer.begin_transaction()
meta = await producer.send_and_wait(
self.topic, b'Hello from transaction', partition=0)
meta2 = await producer2.send_and_wait(
self.topic, b'Hello from non-transaction', partition=0)
# The transaction blocked consumption
task = self.loop.create_task(consumer.getone())
await asyncio.sleep(1, loop=self.loop)
self.assertFalse(task.done())
tp = TopicPartition(self.topic, 0)
self.assertEqual(consumer.last_stable_offset(tp), 0)
self.assertEqual(consumer.highwater(tp), 2)
await producer.commit_transaction()
# Order should be preserved. We first yield the first message, although
# it belongs to a committed afterwards transaction
msg = await task
self.assertEqual(msg.offset, meta.offset)
self.assertEqual(msg.timestamp, meta.timestamp)
self.assertEqual(msg.value, b"Hello from transaction")
self.assertEqual(msg.key, None)
msg = await consumer.getone()
self.assertEqual(msg.offset, meta2.offset)
self.assertEqual(msg.timestamp, meta2.timestamp)
self.assertEqual(msg.value, b"Hello from non-transaction")
self.assertEqual(msg.key, None)
# 3, because we have a commit marker also
tp = TopicPartition(self.topic, 0)
self.assertEqual(consumer.last_stable_offset(tp), 3)
self.assertEqual(consumer.highwater(tp), 3)
async def test_on_partitions_assigned(self, *, handler, thread):
await handler.on_partitions_assigned([
TopicPartition('A', 0),
TopicPartition('B', 3),
])
thread.on_partitions_assigned.assert_called_once_with({
TP('A', 0), TP('B', 3),
})
async def test_is_assigned(subscription_state):
tp1 = TopicPartition("topic", 0)
tp2 = TopicPartition("topic", 1)
assert not subscription_state.is_assigned(tp1)
subscription_state.subscribe({"topic"})
assert not subscription_state.is_assigned(tp1)
subscription_state.assign_from_subscribed({tp1})
assert subscription_state.is_assigned(tp1)
assert not subscription_state.is_assigned(tp2)
def test_is_assigned(subscription_state):
tp1 = TopicPartition("topic", 0)
tp2 = TopicPartition("topic", 1)
assert not subscription_state.is_assigned(tp1)
subscription_state.subscribe(set(["topic"]))
assert not subscription_state.is_assigned(tp1)
subscription_state.assign_from_subscribed(set([tp1]))
assert subscription_state.is_assigned(tp1)
assert not subscription_state.is_assigned(tp2)
def test_coordinator_metadata_update_during_rebalance(self):
# Race condition where client.set_topics start MetadataUpdate, but it
# fails to arrive before leader performed assignment
# Just ensure topics are created
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
yield from client.bootstrap()
yield from self.wait_topic(client, 'topic1')
yield from self.wait_topic(client, 'topic2')
yield from client.close()
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
yield from client.bootstrap()
self.add_cleanup(client.close)
subscription = SubscriptionState('earliest')
client.set_topics(["topic1"])
subscription.subscribe(topics=('topic1', ))
coordinator = GroupCoordinator(
client,
subscription,
loop=self.loop,
group_id='race-rebalance-metadata-update',
heartbeat_interval_ms=20000000)
self.add_cleanup(coordinator.close)
yield from coordinator.ensure_active_group()
# Check that topic's partitions are properly assigned
self.assertEqual(subscription.needs_partition_assignment, False)
self.assertEqual(
set(subscription.assignment.keys()),
{TopicPartition("topic1", 0),
TopicPartition("topic1", 1)})
_metadata_update = client._metadata_update
with mock.patch.object(client, '_metadata_update') as mocked:
@asyncio.coroutine
def _new(*args, **kw):
# Just make metadata updates a bit more slow for test
# robustness
yield from asyncio.sleep(0.5, loop=self.loop)
res = yield from _metadata_update(*args, **kw)
return res
mocked.side_effect = _new
subscription.subscribe(topics=('topic2', ))
client.set_topics(('topic2', ))
yield from coordinator.ensure_active_group()
self.assertEqual(subscription.needs_partition_assignment, False)
self.assertEqual(
set(subscription.assignment.keys()),
{TopicPartition("topic2", 0),
TopicPartition("topic2", 1)})
def test_seek(subscription_state):
tp = TopicPartition("topic1", 0)
tp2 = TopicPartition("topic2", 0)
subscription_state.assign_from_user({tp, tp2})
assignment = subscription_state.subscription.assignment
assert assignment.state_value(tp) is not None
assert not assignment.state_value(tp).has_valid_position
assert assignment.state_value(tp)._position is None
subscription_state.seek(tp, 1000)
assert assignment.state_value(tp).position == 1000
def test_assigned_state(subscription_state):
tp1 = TopicPartition("topic", 0)
tp2 = TopicPartition("topic", 1)
subscription_state.assign_from_user(set([tp1]))
with pytest.raises(IllegalStateError):
subscription_state._assigned_state(tp2)
tp_state = subscription_state._assigned_state(tp1)
assert tp_state is not None
assert repr(tp_state) == \
"TopicPartitionState<Status=PartitionStatus.ASSIGNED position=None>"
async def test_sender__do_txn_offset_commit_not_ok(self):
sender = await self._setup_sender()
offsets = {
TopicPartition("topic", 0): OffsetAndMetadata(10, ""),
TopicPartition("topic", 1): OffsetAndMetadata(11, ""),
}
add_handler = TxnOffsetCommitHandler(sender, offsets, "some_group")
tm = sender._txn_manager
tm.offset_committed = mock.Mock()
def create_response(error_type):
cls = TxnOffsetCommitResponse[0]
resp = cls(throttle_time_ms=300,
errors=[("topic", [(0, error_type.errno),
(1, error_type.errno)])])
return resp
# Handle coordination errors
for error_cls in [
CoordinatorNotAvailableError, NotCoordinatorError,
RequestTimedOutError
]:
with mock.patch.object(sender, "_coordinator_dead") as mocked:
resp = create_response(error_cls)
backoff = add_handler.handle_response(resp)
self.assertEqual(backoff, 0.1)
tm.offset_committed.assert_not_called()
mocked.assert_called_with(CoordinationType.GROUP)
# Not coordination retriable errors
for error_cls in [
CoordinatorLoadInProgressError, UnknownTopicOrPartitionError
]:
resp = create_response(error_cls)
backoff = add_handler.handle_response(resp)
self.assertEqual(backoff, 0.1)
tm.offset_committed.assert_not_called()
# ProducerFenced case
resp = create_response(InvalidProducerEpoch)
with self.assertRaises(ProducerFenced):
add_handler.handle_response(resp)
tm.offset_committed.assert_not_called()
# Handle unknown error
resp = create_response(UnknownError)
with self.assertRaises(UnknownError):
add_handler.handle_response(resp)
tm.offset_committed.assert_not_called()
async def test_assigned_state(subscription_state):
tp1 = TopicPartition("topic", 0)
tp2 = TopicPartition("topic", 1)
subscription_state.assign_from_user({tp1})
with pytest.raises(IllegalStateError):
subscription_state._assigned_state(tp2)
tp_state = subscription_state._assigned_state(tp1)
assert tp_state is not None
assert repr(tp_state) == (
"TopicPartitionState<Status=PartitionStatus.AWAITING_RESET"
" position=None>"
)
async def test_compacted_topic_consumption(self):
# Compacted topics can have offsets skipped
client = AIOKafkaClient(
loop=self.loop,
bootstrap_servers=[])
client.ready = mock.MagicMock()
client.ready.side_effect = asyncio.coroutine(lambda a: True)
client.force_metadata_update = mock.MagicMock()
client.force_metadata_update.side_effect = asyncio.coroutine(
lambda: False)
client.send = mock.MagicMock()
subscriptions = SubscriptionState(loop=self.loop)
fetcher = Fetcher(client, subscriptions, loop=self.loop)
tp = TopicPartition('test', 0)
req = FetchRequest(
-1, # replica_id
100, 100, [(tp.topic, [(tp.partition, 155, 100000)])])
builder = LegacyRecordBatchBuilder(
magic=1, compression_type=0, batch_size=99999999)
builder.append(160, value=b"12345", key=b"1", timestamp=None)
builder.append(162, value=b"23456", key=b"2", timestamp=None)
builder.append(167, value=b"34567", key=b"3", timestamp=None)
batch = bytes(builder.build())
resp = FetchResponse(
[('test', [(
0, 0, 3000, # partition, error_code, highwater_offset
batch # Batch raw bytes
)])])
subscriptions.assign_from_user({tp})
assignment = subscriptions.subscription.assignment
tp_state = assignment.state_value(tp)
client.send.side_effect = asyncio.coroutine(lambda n, r: resp)
tp_state.seek(155)
fetcher._in_flight.add(0)
needs_wake_up = await fetcher._proc_fetch_request(
assignment, 0, req)
self.assertEqual(needs_wake_up, True)
buf = fetcher._records[tp]
# Test successful getone, the closest in batch offset=160
first = buf.getone()
self.assertEqual(tp_state.position, 161)
self.assertEqual(
(first.value, first.key, first.offset),
(b"12345", b"1", 160))
# Test successful getmany
second, third = buf.getall()
self.assertEqual(tp_state.position, 168)
self.assertEqual(
(second.value, second.key, second.offset),
(b"23456", b"2", 162))
self.assertEqual(
(third.value, third.key, third.offset),
(b"34567", b"3", 167))
async def process_next_batch(topic_partition, count):
tp = [topic_partition] if topic_partition else []
read_count = 0
if count == 1:
msg = await client.getone(*tp)
if topic_partition is None:
topic_partition = TopicPartition(msg.topic, msg.partition)
topic = topics[topic_partition.topic]
decoded_msg = topic.decode(msg.value)
topic.partitions[topic_partition].observer.on_next(decoded_msg)
read_count += 1
else:
data = await client.getmany(*tp,
timeout_ms=5000,
max_records=count)
if len(data) > 0:
msgs = data[topic_partition]
topic = topics[topic_partition.topic]
for msg in msgs:
decoded_msg = topic.decode(msg.value)
topic.partitions[topic_partition].observer.on_next(
decoded_msg)
read_count += 1
return read_count
def test_coordinator_ensure_active_group_on_expired_membership(self):
# Do not fail ensure_active_group() if group membership has expired
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
yield from client.bootstrap()
yield from self.wait_topic(client, 'topic1')
subscription = SubscriptionState('earliest')
subscription.subscribe(topics=('topic1', ))
coordinator = GroupCoordinator(client,
subscription,
loop=self.loop,
group_id='test-offsets-group')
yield from coordinator.ensure_active_group()
# during OffsetCommit, UnknownMemberIdError is raised
offsets = {TopicPartition('topic1', 0): OffsetAndMetadata(1, '')}
with mock.patch('aiokafka.errors.for_code') as mocked:
mocked.return_value = Errors.UnknownMemberIdError
with self.assertRaises(Errors.UnknownMemberIdError):
yield from coordinator.commit_offsets(offsets)
self.assertEqual(subscription.needs_partition_assignment, True)
# same exception is raised during ensure_active_group()'s call to
# commit_offsets() via _on_join_prepare() but doesn't break this method
with mock.patch.object(coordinator, "commit_offsets") as mocked:
@asyncio.coroutine
def mock_commit_offsets(*args, **kwargs):
raise Errors.UnknownMemberIdError()
mocked.side_effect = mock_commit_offsets
yield from coordinator.ensure_active_group()
yield from coordinator.close()
yield from client.close()
async def test_sender__produce_request_not_ok(self):
sender = await self._setup_sender()
tp = TopicPartition("my_topic", 0)
batch_mock = mock.Mock()
send_handler = SendProduceReqHandler(sender, {tp: batch_mock})
def create_response(error_type):
cls = ProduceResponse[4]
resp = cls(throttle_time_ms=300,
topics=[("my_topic", [(0, error_type.errno, 0, -1)])])
return resp
# Special case for DuplicateSequenceNumber
resp = create_response(DuplicateSequenceNumber)
send_handler.handle_response(resp)
batch_mock.done.assert_called_with(0, -1, None)
batch_mock.failure.assert_not_called()
self.assertEqual(send_handler._to_reenqueue, [])
batch_mock.reset_mock()
# Special case for InvalidProducerEpoch
resp = create_response(InvalidProducerEpoch)
send_handler.handle_response(resp)
batch_mock.done.assert_not_called()
self.assertNotEqual(batch_mock.failure.call_count, 0)
self.assertEqual(send_handler._to_reenqueue, [])
async def test_sender__do_txn_offset_commit_create(self):
sender = await self._setup_sender()
offsets = {
TopicPartition("topic", 0): OffsetAndMetadata(10, ""),
TopicPartition("topic", 1): OffsetAndMetadata(11, ""),
}
add_handler = TxnOffsetCommitHandler(sender, offsets, "some_group")
req = add_handler.create_request()
self.assertEqual(req.API_KEY, TxnOffsetCommitRequest[0].API_KEY)
self.assertEqual(req.API_VERSION, 0)
self.assertEqual(req.transactional_id, "test_tid")
self.assertEqual(req.group_id, "some_group")
self.assertEqual(req.producer_id, 120)
self.assertEqual(req.producer_epoch, 22)
self.assertEqual(req.topics, [("topic", [(0, 10, ""), (1, 11, "")])])
async def send_batch(self, batch, topic, *, partition):
"""Submit a BatchBuilder for publication.
Arguments:
batch (BatchBuilder): batch object to be published.
topic (str): topic where the batch will be published.
partition (int): partition where this batch will be published.
Returns:
asyncio.Future: object that will be set when the batch is
delivered.
"""
# first make sure the metadata for the topic is available
await self.client._wait_on_metadata(topic)
# We only validate we have the partition in the metadata here
partition = self._partition(topic, partition, None, None, None, None)
# Ensure transaction is started and not committing
if self._txn_manager is not None:
txn_manager = self._txn_manager
if txn_manager.transactional_id is not None and \
not self._txn_manager.is_in_transaction():
raise IllegalOperation(
"Can't send messages while not in transaction")
tp = TopicPartition(topic, partition)
log.debug("Sending batch to %s", tp)
future = await self._message_accumulator.add_batch(
batch, tp, self._request_timeout_ms / 1000)
return future
def handle_response(self, resp):
txn_manager = self._sender._txn_manager
group_id = self._group_id
for topic, partitions in resp.errors:
for partition, error_code in partitions:
tp = TopicPartition(topic, partition)
error_type = Errors.for_code(error_code)
if error_type is Errors.NoError:
offset = self._offsets[tp].offset
log.debug(
"Offset %s for partition %s committed to group %s",
offset, tp, group_id)
txn_manager.offset_committed(tp, offset, group_id)
elif (error_type is CoordinatorNotAvailableError
or error_type is NotCoordinatorError or
# Copied from Java. Not sure why it's only in this case
error_type is RequestTimedOutError):
self._sender._coordinator_dead(CoordinationType.GROUP)
return self._default_backoff
elif (error_type is CoordinatorLoadInProgressError
or error_type is UnknownTopicOrPartitionError):
# We will just retry after backoff
return self._default_backoff
elif error_type is InvalidProducerEpoch:
raise ProducerFenced()
else:
log.error(
"Could not commit offset for partition %s due to "
"unexpected error: %s", partition, error_type)
raise error_type()
def test_subscribe_topic(subscription_state):
mock_listener = MockListener()
subscription_state.subscribe({"tp1", "tp2"}, listener=mock_listener)
assert subscription_state.subscription is not None
assert subscription_state.subscription.topics == {"tp1", "tp2"}
assert subscription_state.subscription.assignment is None
assert subscription_state.subscription.active is True
assert subscription_state.subscription.unsubscribe_future.done() is False
# After subscription to topic we can't change the subscription to pattern
# or user assignment
with pytest.raises(IllegalStateError):
subscription_state.subscribe_pattern(
pattern=re.compile("^tests-.*$"), listener=mock_listener)
with pytest.raises(IllegalStateError):
subscription_state.assign_from_user([TopicPartition("topic", 0)])
# Subsciption of the same type can be applied
old_subsciption = subscription_state.subscription
subscription_state.subscribe(
{"tp1", "tp2", "tp3"}, listener=mock_listener)
assert subscription_state.subscription is not None
assert subscription_state.subscription.topics == {"tp1", "tp2", "tp3"}
assert old_subsciption is not subscription_state.subscription
assert old_subsciption.active is False
assert old_subsciption.unsubscribe_future.done() is True
def test_assigned_partitions(subscription_state):
assert subscription_state.assigned_partitions() == set([])
subscription_state.subscribe(topics=set(["tp1"]))
assert subscription_state.assigned_partitions() == set([])
assignment = {TopicPartition("tp1", 0)}
subscription_state.assign_from_subscribed(assignment)
assert subscription_state.assigned_partitions() == assignment
def test_fetch_result_and_error(loop):
# Add some data
messages = [
ConsumerRecord(topic="some_topic",
partition=1,
offset=0,
timestamp=0,
timestamp_type=0,
key=None,
value=b"some",
checksum=None,
serialized_key_size=0,
serialized_value_size=4)
]
result = FetchResult(TopicPartition("test", 0),
assignment=mock.Mock(),
loop=loop,
message_iterator=iter(messages),
backoff=0,
fetch_offset=0)
assert repr(result) == "<FetchResult position=0>"
error = FetchError(loop=loop, error=OffsetOutOfRangeError({}), backoff=0)
# Python3.7 got rid of trailing comma in exceptions, which makes the line
# diffrent between 3.6 and 3.7.
assert repr(error) in [
"<FetchError error=OffsetOutOfRangeError({},)>",
"<FetchError error=OffsetOutOfRangeError({})>"
]
async def send(self, topic, value=None, key=None, partition=None,
timestamp_ms=None):
"""Publish a message to a topic.
Arguments:
topic (str): topic where the message will be published
value (optional): message value. Must be type bytes, or be
serializable to bytes via configured value_serializer. If value
is None, key is required and message acts as a 'delete'.
See kafka compaction documentation for more details:
http://kafka.apache.org/documentation.html#compaction
(compaction requires kafka >= 0.8.1)
partition (int, optional): optionally specify a partition. If not
set, the partition will be selected using the configured
'partitioner'.
key (optional): a key to associate with the message. Can be used to
determine which partition to send the message to. If partition
is None (and producer's partitioner config is left as default),
then messages with the same key will be delivered to the same
partition (but if key is None, partition is chosen randomly).
Must be type bytes, or be serializable to bytes via configured
key_serializer.
timestamp_ms (int, optional): epoch milliseconds (from Jan 1 1970
UTC) to use as the message timestamp. Defaults to current time.
Returns:
asyncio.Future: object that will be set when message is
processed
Raises:
kafka.KafkaTimeoutError: if we can't schedule this record (
pending buffer is full) in up to `request_timeout_ms`
milliseconds.
Note:
The returned future will wait based on `request_timeout_ms`
setting. Cancelling the returned future **will not** stop event
from being sent, but cancelling the ``send`` coroutine itself
**will**.
"""
assert value is not None or self.client.api_version >= (0, 8, 1), (
'Null messages require kafka >= 0.8.1')
assert not (value is None and key is None), \
'Need at least one: key or value'
# first make sure the metadata for the topic is available
await self.client._wait_on_metadata(topic)
key_bytes, value_bytes = self._serialize(topic, key, value)
partition = self._partition(topic, partition, key, value,
key_bytes, value_bytes)
tp = TopicPartition(topic, partition)
log.debug("Sending (key=%s value=%s) to %s", key, value, tp)
fut = await self._message_accumulator.add_message(
tp, key_bytes, value_bytes, self._request_timeout_ms / 1000,
timestamp_ms=timestamp_ms)
return fut
async def test_sender__do_add_partitions_to_txn_create(self):
sender = await self._setup_sender()
tps = [
TopicPartition("topic", 0),
TopicPartition("topic", 1),
TopicPartition("topic2", 1)
]
add_handler = AddPartitionsToTxnHandler(sender, tps)
req = add_handler.create_request()
self.assertEqual(req.API_KEY, AddPartitionsToTxnRequest[0].API_KEY)
self.assertEqual(req.API_VERSION, 0)
self.assertEqual(req.transactional_id, "test_tid")
self.assertEqual(req.producer_id, 120)
self.assertEqual(req.producer_epoch, 22)
self.assertEqual(list(sorted(req.topics)),
list(sorted([("topic", [0, 1]), ("topic2", [1])])))
def test_assignment(self, *, cthread, _consumer):
cthread._consumer = _consumer
_consumer.assignment.return_value = {
TopicPartition(TP1.topic, TP1.partition),
}
assignment = cthread.assignment()
assert assignment == {TP1}
assert all(isinstance(x, TP) for x in assignment)
def handle_response(self, response):
for topic, partitions in response.topics:
for partition_info in partitions:
global_error = None
log_start_offset = None
if response.API_VERSION < 2:
partition, error_code, offset = partition_info
# Mimic CREATE_TIME to take user provided timestamp
timestamp = -1
elif 2 <= response.API_VERSION <= 4:
partition, error_code, offset, timestamp = partition_info
elif 5 <= response.API_VERSION <= 7:
(
partition, error_code, offset, timestamp,
log_start_offset
) = partition_info
else:
# the ignored parameter is record_error of type
# list[(batch_index: int, error_message: str)]
(
partition, error_code, offset, timestamp,
log_start_offset, _, global_error
) = partition_info
tp = TopicPartition(topic, partition)
error = Errors.for_code(error_code)
batch = self._batches.get(tp)
if batch is None:
continue
if error is Errors.NoError:
batch.done(offset, timestamp, log_start_offset)
elif error is DuplicateSequenceNumber:
# If we have received a duplicate sequence error,
# it means that the sequence number has advanced
# beyond the sequence of the current batch, and we
# haven't retained batch metadata on the broker to
# return the correct offset and timestamp.
#
# The only thing we can do is to return success to
# the user and not return a valid offset and
# timestamp.
batch.done(offset, timestamp, log_start_offset)
elif not self._can_retry(error(), batch):
if error is InvalidProducerEpoch:
exc = ProducerFenced()
elif error is TopicAuthorizationFailedError:
exc = error(topic)
else:
exc = error()
batch.failure(exception=exc)
else:
log.warning(
"Got error produce response on topic-partition"
" %s, retrying. Error: %s", tp, global_error or error)
# Ok, we can retry this batch
if getattr(error, "invalid_metadata", False):
self._client.force_metadata_update()
self._to_reenqueue.append(batch)
def test_offsets_failed_scenarios(self):
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
yield from client.bootstrap()
yield from self.wait_topic(client, 'topic1')
subscription = SubscriptionState('earliest')
subscription.subscribe(topics=('topic1', ))
coordinator = GroupCoordinator(client,
subscription,
loop=self.loop,
group_id='test-offsets-group')
yield from coordinator.ensure_active_group()
offsets = {TopicPartition('topic1', 0): OffsetAndMetadata(1, '')}
yield from coordinator.commit_offsets(offsets)
with mock.patch('aiokafka.errors.for_code') as mocked:
mocked.return_value = Errors.GroupAuthorizationFailedError
with self.assertRaises(Errors.GroupAuthorizationFailedError):
yield from coordinator.commit_offsets(offsets)
mocked.return_value = Errors.TopicAuthorizationFailedError
with self.assertRaises(Errors.TopicAuthorizationFailedError):
yield from coordinator.commit_offsets(offsets)
mocked.return_value = Errors.InvalidCommitOffsetSizeError
with self.assertRaises(Errors.InvalidCommitOffsetSizeError):
yield from coordinator.commit_offsets(offsets)
mocked.return_value = Errors.GroupLoadInProgressError
with self.assertRaises(Errors.GroupLoadInProgressError):
yield from coordinator.commit_offsets(offsets)
mocked.return_value = Errors.RebalanceInProgressError
with self.assertRaises(Errors.RebalanceInProgressError):
yield from coordinator.commit_offsets(offsets)
self.assertEqual(subscription.needs_partition_assignment, True)
subscription.needs_partition_assignment = False
mocked.return_value = Errors.UnknownMemberIdError
with self.assertRaises(Errors.UnknownMemberIdError):
yield from coordinator.commit_offsets(offsets)
self.assertEqual(subscription.needs_partition_assignment, True)
mocked.return_value = KafkaError
with self.assertRaises(KafkaError):
yield from coordinator.commit_offsets(offsets)
mocked.return_value = Errors.NotCoordinatorForGroupError
with self.assertRaises(Errors.NotCoordinatorForGroupError):
yield from coordinator.commit_offsets(offsets)
self.assertEqual(coordinator.coordinator_id, None)
with self.assertRaises(Errors.GroupCoordinatorNotAvailableError):
yield from coordinator.commit_offsets(offsets)
yield from coordinator.close()
yield from client.close()
