def fetch_all_metadata(self):
cluster_md = ClusterMetadata(
metadata_max_age_ms=self._metadata_max_age_ms)
updated = yield from self._metadata_update(cluster_md, [])
if not updated:
raise KafkaError(
'Unable to get cluster metadata over all known brokers')
return cluster_md
def test_commit_message_error(self, config):
with mock_kafka() as (mock_client, mock_consumer):
consumer = KafkaSimpleConsumer('test_topic', config)
consumer.connect()
mock_client.return_value.send_offset_commit_request \
.side_effect = KafkaError("Boom!")
actual = consumer.commit_message(
Message(0, 100, 'mykey', 'myvalue'), )
assert actual is False
def test_failed_sync_group(self):
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
subscription = SubscriptionState('latest')
subscription.subscribe(topics=('topic1', ))
coordinator = GroupCoordinator(client,
subscription,
loop=self.loop,
heartbeat_interval_ms=20000)
@asyncio.coroutine
def do_sync_group():
rebalance = CoordinatorGroupRebalance(
coordinator,
coordinator.group_id,
coordinator.coordinator_id,
subscription.subscription,
coordinator._assignors,
coordinator._session_timeout_ms,
coordinator._retry_backoff_ms,
loop=self.loop)
yield from rebalance._on_join_follower()
with self.assertRaises(GroupCoordinatorNotAvailableError):
yield from do_sync_group()
mocked = mock.MagicMock()
coordinator._client = mocked
coordinator.member_id = 'some_invalid_member_id'
coordinator.coordinator_unknown = asyncio.coroutine(lambda: False)
mocked.send.side_effect = Errors.UnknownMemberIdError()
with self.assertRaises(Errors.UnknownMemberIdError):
yield from do_sync_group()
self.assertEqual(coordinator.member_id,
JoinGroupRequest.UNKNOWN_MEMBER_ID)
mocked.send.side_effect = Errors.NotCoordinatorForGroupError()
coordinator.coordinator_id = 'some_id'
with self.assertRaises(Errors.NotCoordinatorForGroupError):
yield from do_sync_group()
self.assertEqual(coordinator.coordinator_id, None)
mocked.send.side_effect = KafkaError()
with self.assertRaises(KafkaError):
yield from do_sync_group()
# client sends LeaveGroupRequest to group coordinator
# if generation > 0 (means that client is a member of group)
# expecting no exception in this case (error should be ignored in close
# method)
coordinator.generation = 33
yield from coordinator.close()
def test_metadata_update_fail(self):
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
yield from client.bootstrap()
with mock.patch.object(AIOKafkaConnection, 'send') as mocked:
mocked.side_effect = KafkaError('mocked exception')
updated = yield from client.force_metadata_update()
self.assertEqual(updated, False)
with self.assertRaises(KafkaError):
yield from client.fetch_all_metadata()
async def test_metadata_update_fail(self):
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
await client.bootstrap()
# Make sure the connection is initialize before mock to avoid crashing
# api_version routine
await client.force_metadata_update()
with mock.patch.object(AIOKafkaConnection, 'send') as mocked:
mocked.side_effect = KafkaError('mocked exception')
updated = await client.force_metadata_update()
self.assertEqual(updated, False)
with self.assertRaises(KafkaError):
await client.fetch_all_metadata()
await client.close()
def retry_if_failed(fn, *args, retry_limit = 5, retry_wait_s = 1, **kwargs):
retry_attempts = 0
while retry_attempts < retry_limit:
try:
with kafka_client_pool.kafka_client() as client:
call_result = fn(client, *args, **kwargs)
if retry_attempts > 0:
logging.info('[#KFKRETRY] Made %s attempts to make kafka call', retry_attempts)
return call_result
except KafkaError:
logging.error('[#KFKFAIL] Error on kafka communication. Will reset connection to retry after waiting %s s', retry_wait_s)
logging.error(traceback.format_exc())
sleep(retry_wait_s)
retry_attempts += 1
raise KafkaError()
def test_check_version(self):
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
yield from client.bootstrap()
ver = yield from client.check_version()
self.assertTrue('0.' in ver)
yield from self.wait_topic(client, 'some_test_topic')
ver2 = yield from client.check_version()
self.assertEqual(ver, ver2)
ver2 = yield from client.check_version(client.get_random_node())
self.assertEqual(ver, ver2)
with mock.patch.object(AIOKafkaConnection, 'send') as mocked:
mocked.side_effect = KafkaError('mocked exception')
with self.assertRaises(UnrecognizedBrokerVersion):
yield from client.check_version(client.get_random_node())
client._get_conn = asyncio.coroutine(lambda _: None)
with self.assertRaises(ConnectionError):
yield from client.check_version()
async def test_check_version(self):
kafka_version = tuple(int(x) for x in self.kafka_version.split("."))
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
await client.bootstrap()
ver = await client.check_version()
self.assertEqual(kafka_version[:2], ver[:2])
await self.wait_topic(client, 'some_test_topic')
ver2 = await client.check_version()
self.assertEqual(ver, ver2)
ver2 = await client.check_version(client.get_random_node())
self.assertEqual(ver, ver2)
with mock.patch.object(AIOKafkaConnection, 'send') as mocked:
mocked.side_effect = KafkaError('mocked exception')
with self.assertRaises(UnrecognizedBrokerVersion):
await client.check_version(client.get_random_node())
client._get_conn = asyncio.coroutine(lambda _, **kw: None)
with self.assertRaises(ConnectionError):
await client.check_version()
await client.close()
def test_failed_bootstrap(self):
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
with mock.patch.object(AIOKafkaConnection, 'send') as mock_send:
mock_send.side_effect = KafkaError('some kafka error')
with self.assertRaises(ConnectionError):
yield from client.bootstrap()
def test_notify_with_retry(self, fake_send, fake_ensure_connection):
conn = self.driver._get_connection(kafka_driver.PURPOSE_SEND)
fake_send.side_effect = KafkaError("fake_exception")
conn.notify_send("fake_topic", {"fake_ctxt": "fake_param"},
{"fake_text": "fake_message_2"}, 10)
self.assertEqual(10, len(fake_send.mock_calls))
