def test_different_connections_in_conn_groups(self):
client = AIOKafkaClient(loop=self.loop,
bootstrap_servers=self.hosts,
metadata_max_age_ms=10000)
yield from client.bootstrap()
self.add_cleanup(client.close)
node_id = client.get_random_node()
conn1 = yield from client._get_conn(node_id)
conn2 = yield from client._get_conn(node_id,
group=ConnectionGroup.COORDINATION)
self.assertTrue(conn1 is not conn2)
self.assertEqual((conn1.host, conn1.port), (conn2.host, conn2.port))
async def test_check_version(self):
kafka_version = tuple(int(x) for x in self.kafka_version.split("."))
client = AIOKafkaClient(bootstrap_servers=self.hosts)
await client.bootstrap()
ver = await client.check_version()
expected_version = kafka_version[:2]
# No significant protocol changed, no way to differencieate
if expected_version == (2, 2):
expected_version = (2, 1)
elif expected_version == (2, 4):
expected_version = (2, 3)
self.assertEqual(expected_version, ver[:2])
await self.wait_topic(client, 'some_test_topic')
ver2 = await client.check_version()
self.assertEqual(ver, ver2)
ver2 = await client.check_version(client.get_random_node())
self.assertEqual(ver, ver2)
with mock.patch.object(AIOKafkaConnection, 'send') as mocked:
mocked.side_effect = KafkaError('mocked exception')
with self.assertRaises(UnrecognizedBrokerVersion):
await client.check_version(client.get_random_node())
async def _get_conn(*args: Any, **kw: Any):
return None
client._get_conn = _get_conn
with self.assertRaises(KafkaConnectionError):
await client.check_version()
await client.close()
async def test_client_receive_zero_brokers_timeout_on_send(self):
brokers = [(0, 'broker_1', 4567), (1, 'broker_2', 5678)]
correct_meta = MetadataResponse(brokers, [])
async def send(*args, **kwargs):
raise asyncio.TimeoutError()
client = AIOKafkaClient(bootstrap_servers=['broker_1:4567'],
api_version="0.10")
conn = mock.Mock()
client._conns = [mock.Mock()]
async def _get_conn(*args: Any, **kwargs: Any):
return conn
client._get_conn = mock.Mock()
client._get_conn.side_effect = _get_conn
conn.send = mock.Mock()
conn.send.side_effect = send
client.cluster.update_metadata(correct_meta)
brokers_before = client.cluster.brokers()
await client._metadata_update(client.cluster, [])
# There broker list should not be purged
self.assertNotEqual(client.cluster.brokers(), set([]))
self.assertEqual(client.cluster.brokers(), brokers_before)
async def test_client_receive_zero_brokers(self):
brokers = [(0, 'broker_1', 4567), (1, 'broker_2', 5678)]
correct_meta = MetadataResponse(brokers, [])
bad_response = MetadataResponse([], [])
async def send(*args, **kwargs):
return bad_response
client = AIOKafkaClient(loop=self.loop,
bootstrap_servers=['broker_1:4567'],
api_version="0.10")
conn = mock.Mock()
client._conns = [mock.Mock()]
client._get_conn = mock.Mock()
client._get_conn.side_effect = asyncio.coroutine(lambda x: conn)
conn.send = mock.Mock()
conn.send.side_effect = send
client.cluster.update_metadata(correct_meta)
brokers_before = client.cluster.brokers()
await client._metadata_update(client.cluster, [])
# There broker list should not be purged
self.assertNotEqual(client.cluster.brokers(), set([]))
self.assertEqual(client.cluster.brokers(), brokers_before)
def test_client_receive_zero_brokers(self):
brokers = [
(0, 'broker_1', 4567),
(1, 'broker_2', 5678)
]
correct_meta = MetadataResponse(brokers, [])
bad_response = MetadataResponse([], [])
@asyncio.coroutine
def send(*args, **kwargs):
return bad_response
client = AIOKafkaClient(loop=self.loop,
bootstrap_servers=['broker_1:4567'],
api_version="0.10")
conn = mock.Mock()
client._conns = [mock.Mock()]
client._get_conn = mock.Mock()
client._get_conn.side_effect = asyncio.coroutine(lambda x: conn)
conn.send = mock.Mock()
conn.send.side_effect = send
client.cluster.update_metadata(correct_meta)
brokers_before = client.cluster.brokers()
yield from client._metadata_update(client.cluster, [])
# There broker list should not be purged
self.assertNotEqual(client.cluster.brokers(), set([]))
self.assertEqual(client.cluster.brokers(), brokers_before)
def test_different_connections_in_conn_groups(self):
client = AIOKafkaClient(
loop=self.loop,
bootstrap_servers=self.hosts,
metadata_max_age_ms=10000)
yield from client.bootstrap()
self.add_cleanup(client.close)
node_id = client.get_random_node()
broker = client.cluster.broker_metadata(node_id)
client.cluster.add_coordinator(
node_id, broker.host, broker.port, rack=None,
purpose=(CoordinationType.GROUP, ""))
conn1 = yield from client._get_conn(node_id)
conn2 = yield from client._get_conn(
node_id, group=ConnectionGroup.COORDINATION)
self.assertTrue(conn1 is not conn2)
self.assertEqual((conn1.host, conn1.port), (conn2.host, conn2.port))
def test_different_connections_in_conn_groups(self):
client = AIOKafkaClient(loop=self.loop,
bootstrap_servers=self.hosts,
metadata_max_age_ms=10000)
yield from client.bootstrap()
self.add_cleanup(client.close)
node_id = client.get_random_node()
broker = client.cluster.broker_metadata(node_id)
client.cluster.add_coordinator(node_id,
broker.host,
broker.port,
rack=None,
purpose=(CoordinationType.GROUP, ""))
conn1 = yield from client._get_conn(node_id)
conn2 = yield from client._get_conn(node_id,
group=ConnectionGroup.COORDINATION)
self.assertTrue(conn1 is not conn2)
self.assertEqual((conn1.host, conn1.port), (conn2.host, conn2.port))
def test_send_timeout_deletes_connection(self):
correct_response = MetadataResponse([], [])
@asyncio.coroutine
def send_exception(*args, **kwargs):
raise asyncio.TimeoutError()
@asyncio.coroutine
def send(*args, **kwargs):
return correct_response
@asyncio.coroutine
def get_conn(self, node_id, *, group=0):
conn_id = (node_id, group)
if conn_id in self._conns:
conn = self._conns[conn_id]
if not conn.connected():
del self._conns[conn_id]
else:
return conn
conn = mock.MagicMock()
conn.send.side_effect = send
self._conns[conn_id] = conn
return conn
node_id = 0
conn = mock.MagicMock()
conn.send.side_effect = send_exception
conn.connected.return_value = True
mocked_conns = {(node_id, 0): conn}
client = AIOKafkaClient(loop=self.loop,
bootstrap_servers=['broker_1:4567'])
client._conns = mocked_conns
client._get_conn = types.MethodType(get_conn, client)
# first send timeouts
with self.assertRaises(RequestTimedOutError):
yield from client.send(0, MetadataRequest([]))
conn.close.assert_called_once_with(
reason=CloseReason.CONNECTION_TIMEOUT)
# this happens because conn was closed
conn.connected.return_value = False
# second send gets new connection and obtains result
response = yield from client.send(0, MetadataRequest([]))
self.assertEqual(response, correct_response)
self.assertNotEqual(conn, client._conns[(node_id, 0)])
def test_send_timeout_deletes_connection(self):
correct_response = MetadataResponse([], [])
@asyncio.coroutine
def send_exception(*args, **kwargs):
raise asyncio.TimeoutError()
@asyncio.coroutine
def send(*args, **kwargs):
return correct_response
@asyncio.coroutine
def get_conn(self, node_id, *, group=0):
conn_id = (node_id, group)
if conn_id in self._conns:
conn = self._conns[conn_id]
if not conn.connected():
del self._conns[conn_id]
else:
return conn
conn = mock.MagicMock()
conn.send.side_effect = send
self._conns[conn_id] = conn
return conn
node_id = 0
conn = mock.MagicMock()
conn.send.side_effect = send_exception
conn.connected.return_value = True
mocked_conns = {(node_id, 0): conn}
client = AIOKafkaClient(loop=self.loop,
bootstrap_servers=['broker_1:4567'])
client._conns = mocked_conns
client._get_conn = types.MethodType(get_conn, client)
# first send timeouts
with self.assertRaises(RequestTimedOutError):
yield from client.send(0, MetadataRequest([]))
conn.close.assert_called_once_with(
reason=CloseReason.CONNECTION_TIMEOUT)
# this happens because conn was closed
conn.connected.return_value = False
# second send gets new connection and obtains result
response = yield from client.send(0, MetadataRequest([]))
self.assertEqual(response, correct_response)
self.assertNotEqual(conn, client._conns[(node_id, 0)])
def test_check_version(self):
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
yield from client.bootstrap()
ver = yield from client.check_version()
self.assertTrue('0.' in ver)
yield from self.wait_topic(client, 'some_test_topic')
ver2 = yield from client.check_version()
self.assertEqual(ver, ver2)
ver2 = yield from client.check_version(client.get_random_node())
self.assertEqual(ver, ver2)
with mock.patch.object(AIOKafkaConnection, 'send') as mocked:
mocked.side_effect = KafkaError('mocked exception')
with self.assertRaises(UnrecognizedBrokerVersion):
yield from client.check_version(client.get_random_node())
client._get_conn = asyncio.coroutine(lambda _: None)
with self.assertRaises(ConnectionError):
yield from client.check_version()
def test_check_version(self):
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
yield from client.bootstrap()
ver = yield from client.check_version()
self.assertTrue('0.' in ver)
yield from self.wait_topic(client, 'some_test_topic')
ver2 = yield from client.check_version()
self.assertEqual(ver, ver2)
ver2 = yield from client.check_version(client.get_random_node())
self.assertEqual(ver, ver2)
with mock.patch.object(
AIOKafkaConnection, 'send') as mocked:
mocked.side_effect = KafkaError('mocked exception')
with self.assertRaises(UnrecognizedBrokerVersion):
yield from client.check_version(client.get_random_node())
client._get_conn = asyncio.coroutine(lambda _: None)
with self.assertRaises(ConnectionError):
yield from client.check_version()
async def test_check_version(self):
kafka_version = tuple(int(x) for x in self.kafka_version.split("."))
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
await client.bootstrap()
ver = await client.check_version()
self.assertEqual(kafka_version[:2], ver[:2])
await self.wait_topic(client, 'some_test_topic')
ver2 = await client.check_version()
self.assertEqual(ver, ver2)
ver2 = await client.check_version(client.get_random_node())
self.assertEqual(ver, ver2)
with mock.patch.object(AIOKafkaConnection, 'send') as mocked:
mocked.side_effect = KafkaError('mocked exception')
with self.assertRaises(UnrecognizedBrokerVersion):
await client.check_version(client.get_random_node())
client._get_conn = asyncio.coroutine(lambda _, **kw: None)
with self.assertRaises(ConnectionError):
await client.check_version()
await client.close()
def test_metadata_updated_on_socket_disconnect(self):
# Related to issue 176. A disconnect means that either we lost
# connection to the node, or we have a node failure. In both cases
# there's a high probability that Leader distribution will also change.
client = AIOKafkaClient(loop=self.loop,
bootstrap_servers=self.hosts,
metadata_max_age_ms=10000)
yield from client.bootstrap()
self.add_cleanup(client.close)
# Init a clonnection
node_id = client.get_random_node()
assert node_id is not None
req = MetadataRequest([])
yield from client.send(node_id, req)
# No metadata update pending atm
self.assertFalse(client._md_update_waiter.done())
# Connection disconnect should trigger an update
conn = yield from client._get_conn(node_id)
conn.close(reason=CloseReason.CONNECTION_BROKEN)
self.assertTrue(client._md_update_waiter.done())
def test_check_version(self):
kafka_version = tuple(int(x) for x in self.kafka_version.split("."))
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
yield from client.bootstrap()
ver = yield from client.check_version()
self.assertEqual(kafka_version[:2], ver[:2])
yield from self.wait_topic(client, 'some_test_topic')
ver2 = yield from client.check_version()
self.assertEqual(ver, ver2)
ver2 = yield from client.check_version(client.get_random_node())
self.assertEqual(ver, ver2)
with mock.patch.object(
AIOKafkaConnection, 'send') as mocked:
mocked.side_effect = KafkaError('mocked exception')
with self.assertRaises(UnrecognizedBrokerVersion):
yield from client.check_version(client.get_random_node())
client._get_conn = asyncio.coroutine(lambda _, **kw: None)
with self.assertRaises(ConnectionError):
yield from client.check_version()
yield from client.close()
def test_metadata_updated_on_socket_disconnect(self):
# Related to issue 176. A disconnect means that either we lost
# connection to the node, or we have a node failure. In both cases
# there's a high probability that Leader distribution will also change.
client = AIOKafkaClient(
loop=self.loop,
bootstrap_servers=self.hosts,
metadata_max_age_ms=10000)
yield from client.bootstrap()
self.add_cleanup(client.close)
# Init a clonnection
node_id = client.get_random_node()
assert node_id is not None
req = MetadataRequest([])
yield from client.send(node_id, req)
# No metadata update pending atm
self.assertFalse(client._md_update_waiter.done())
# Connection disconnect should trigger an update
conn = yield from client._get_conn(node_id)
conn.close(reason=CloseReason.CONNECTION_BROKEN)
self.assertTrue(client._md_update_waiter.done())