def fetch_all_metadata(self): cluster_md = ClusterMetadata( metadata_max_age_ms=self._metadata_max_age_ms) updated = yield from self._metadata_update(cluster_md, []) if not updated: raise KafkaError( 'Unable to get cluster metadata over all known brokers') return cluster_md
def test_commit_message_error(self, config): with mock_kafka() as (mock_client, mock_consumer): consumer = KafkaSimpleConsumer('test_topic', config) consumer.connect() mock_client.return_value.send_offset_commit_request \ .side_effect = KafkaError("Boom!") actual = consumer.commit_message( Message(0, 100, 'mykey', 'myvalue'), ) assert actual is False
def test_failed_sync_group(self): client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) subscription = SubscriptionState('latest') subscription.subscribe(topics=('topic1', )) coordinator = GroupCoordinator(client, subscription, loop=self.loop, heartbeat_interval_ms=20000) @asyncio.coroutine def do_sync_group(): rebalance = CoordinatorGroupRebalance( coordinator, coordinator.group_id, coordinator.coordinator_id, subscription.subscription, coordinator._assignors, coordinator._session_timeout_ms, coordinator._retry_backoff_ms, loop=self.loop) yield from rebalance._on_join_follower() with self.assertRaises(GroupCoordinatorNotAvailableError): yield from do_sync_group() mocked = mock.MagicMock() coordinator._client = mocked coordinator.member_id = 'some_invalid_member_id' coordinator.coordinator_unknown = asyncio.coroutine(lambda: False) mocked.send.side_effect = Errors.UnknownMemberIdError() with self.assertRaises(Errors.UnknownMemberIdError): yield from do_sync_group() self.assertEqual(coordinator.member_id, JoinGroupRequest.UNKNOWN_MEMBER_ID) mocked.send.side_effect = Errors.NotCoordinatorForGroupError() coordinator.coordinator_id = 'some_id' with self.assertRaises(Errors.NotCoordinatorForGroupError): yield from do_sync_group() self.assertEqual(coordinator.coordinator_id, None) mocked.send.side_effect = KafkaError() with self.assertRaises(KafkaError): yield from do_sync_group() # client sends LeaveGroupRequest to group coordinator # if generation > 0 (means that client is a member of group) # expecting no exception in this case (error should be ignored in close # method) coordinator.generation = 33 yield from coordinator.close()
def test_metadata_update_fail(self): client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) yield from client.bootstrap() with mock.patch.object(AIOKafkaConnection, 'send') as mocked: mocked.side_effect = KafkaError('mocked exception') updated = yield from client.force_metadata_update() self.assertEqual(updated, False) with self.assertRaises(KafkaError): yield from client.fetch_all_metadata()
async def test_metadata_update_fail(self): client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) await client.bootstrap() # Make sure the connection is initialize before mock to avoid crashing # api_version routine await client.force_metadata_update() with mock.patch.object(AIOKafkaConnection, 'send') as mocked: mocked.side_effect = KafkaError('mocked exception') updated = await client.force_metadata_update() self.assertEqual(updated, False) with self.assertRaises(KafkaError): await client.fetch_all_metadata() await client.close()
def retry_if_failed(fn, *args, retry_limit = 5, retry_wait_s = 1, **kwargs): retry_attempts = 0 while retry_attempts < retry_limit: try: with kafka_client_pool.kafka_client() as client: call_result = fn(client, *args, **kwargs) if retry_attempts > 0: logging.info('[#KFKRETRY] Made %s attempts to make kafka call', retry_attempts) return call_result except KafkaError: logging.error('[#KFKFAIL] Error on kafka communication. Will reset connection to retry after waiting %s s', retry_wait_s) logging.error(traceback.format_exc()) sleep(retry_wait_s) retry_attempts += 1 raise KafkaError()
def test_check_version(self): client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) yield from client.bootstrap() ver = yield from client.check_version() self.assertTrue('0.' in ver) yield from self.wait_topic(client, 'some_test_topic') ver2 = yield from client.check_version() self.assertEqual(ver, ver2) ver2 = yield from client.check_version(client.get_random_node()) self.assertEqual(ver, ver2) with mock.patch.object(AIOKafkaConnection, 'send') as mocked: mocked.side_effect = KafkaError('mocked exception') with self.assertRaises(UnrecognizedBrokerVersion): yield from client.check_version(client.get_random_node()) client._get_conn = asyncio.coroutine(lambda _: None) with self.assertRaises(ConnectionError): yield from client.check_version()
async def test_check_version(self): kafka_version = tuple(int(x) for x in self.kafka_version.split(".")) client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) await client.bootstrap() ver = await client.check_version() self.assertEqual(kafka_version[:2], ver[:2]) await self.wait_topic(client, 'some_test_topic') ver2 = await client.check_version() self.assertEqual(ver, ver2) ver2 = await client.check_version(client.get_random_node()) self.assertEqual(ver, ver2) with mock.patch.object(AIOKafkaConnection, 'send') as mocked: mocked.side_effect = KafkaError('mocked exception') with self.assertRaises(UnrecognizedBrokerVersion): await client.check_version(client.get_random_node()) client._get_conn = asyncio.coroutine(lambda _, **kw: None) with self.assertRaises(ConnectionError): await client.check_version() await client.close()
def test_failed_bootstrap(self): client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) with mock.patch.object(AIOKafkaConnection, 'send') as mock_send: mock_send.side_effect = KafkaError('some kafka error') with self.assertRaises(ConnectionError): yield from client.bootstrap()
def test_notify_with_retry(self, fake_send, fake_ensure_connection): conn = self.driver._get_connection(kafka_driver.PURPOSE_SEND) fake_send.side_effect = KafkaError("fake_exception") conn.notify_send("fake_topic", {"fake_ctxt": "fake_param"}, {"fake_text": "fake_message_2"}, 10) self.assertEqual(10, len(fake_send.mock_calls))