def check_version(self, node_id=None):
"""Attempt to guess the broker version"""
if node_id is None:
if self._conns:
node_id = list(self._conns.keys())[0]
else:
assert self.cluster.brokers(), 'no brokers in metadata'
node_id = list(self.cluster.brokers())[0].nodeId
from kafka.protocol.admin import (ListGroupsRequest_v0,
ApiVersionRequest_v0)
from kafka.protocol.commit import (OffsetFetchRequest_v0,
GroupCoordinatorRequest_v0)
from kafka.protocol.metadata import MetadataRequest_v0
test_cases = [
('0.10', ApiVersionRequest_v0()),
('0.9', ListGroupsRequest_v0()),
('0.8.2', GroupCoordinatorRequest_v0('aiokafka-default-group')),
('0.8.1', OffsetFetchRequest_v0('aiokafka-default-group', [])),
('0.8.0', MetadataRequest_v0([])),
]
# kafka kills the connection when it doesnt recognize an API request
# so we can send a test request and then follow immediately with a
# vanilla MetadataRequest. If the server did not recognize the first
# request, both will be failed with a ConnectionError that wraps
# socket.error (32, 54, or 104)
conn = yield from self._get_conn(node_id)
if conn is None:
raise ConnectionError(
"No connection to node with id {}".format(node_id))
for version, request in test_cases:
try:
if not conn.connected():
yield from conn.connect()
assert conn, 'no connection to node with id {}'.format(node_id)
# request can be ignored by Kafka broker,
# so we send metadata request and wait response
task = self._loop.create_task(conn.send(request))
yield from asyncio.wait([task], timeout=0.1, loop=self._loop)
try:
yield from conn.send(MetadataRequest_v0([]))
except KafkaError:
# metadata request can be cancelled in case
# of invalid correlationIds order
pass
yield from task
except KafkaError:
continue
else:
# To avoid having a connection in undefined state
if node_id != "bootstrap" and conn.connected():
conn.close()
return version
raise UnrecognizedBrokerVersion()
def check_version(self, node_id=None):
"""Attempt to guess the broker version"""
if node_id is None:
if self._conns:
node_id = list(self._conns.keys())[0]
else:
assert self.cluster.brokers(), 'no brokers in metadata'
node_id = list(self.cluster.brokers())[0].nodeId
from kafka.protocol.admin import ListGroupsRequest_v0
from kafka.protocol.commit import (OffsetFetchRequest_v0,
GroupCoordinatorRequest_v0)
from kafka.protocol.metadata import MetadataRequest_v0
test_cases = [
('0.9', ListGroupsRequest_v0()),
('0.8.2', GroupCoordinatorRequest_v0('aiokafka-default-group')),
('0.8.1', OffsetFetchRequest_v0('aiokafka-default-group', [])),
('0.8.0', MetadataRequest_v0([])),
]
# kafka kills the connection when it doesnt recognize an API request
# so we can send a test request and then follow immediately with a
# vanilla MetadataRequest. If the server did not recognize the first
# request, both will be failed with a ConnectionError that wraps
# socket.error (32, 54, or 104)
conn = yield from self._get_conn(node_id)
if conn is None:
raise ConnectionError(
"No connection to node with id {}".format(node_id))
for version, request in test_cases:
try:
if not conn.connected():
yield from conn.connect()
assert conn, 'no connection to node with id {}'.format(node_id)
yield from conn.send(request)
except KafkaError:
continue
else:
return version
raise UnrecognizedBrokerVersion()
async def wait_no_lag(self):
print("Ensuring no lag")
consumer_group = "f-simple"
client = self.client
await self.client.bootstrap()
source = self.source
source_builder = self._source_builder
source_highwaters = source_builder._highwaters()
source_tps = source_builder._assignment
protocol_tps = [(source, [tp.partition for tp in source_tps])]
node_id = next(broker.nodeId for broker in client.cluster.brokers())
coordinator_request = GroupCoordinatorRequest_v0(consumer_group)
coordinator_response = await client.send(node_id, coordinator_request)
coordinator_id = coordinator_response.coordinator_id
while True:
consumer_offsets_req = OffsetFetchRequest_v1(
consumer_group, protocol_tps)
consumer_offsets_resp = await client.send(coordinator_id,
consumer_offsets_req)
topics = consumer_offsets_resp.topics
assert len(topics) == 1, f"{topics!r}"
topic, partition_resps = topics[0]
assert topic == source, f"{source}"
assert len(partition_resps) == len(source_tps)
# + 1 is to account for the difference in how faust commits
positions = {
TopicPartition(topic=source, partition=partition): offset + 1
for partition, offset, _, _ in partition_resps
}
if positions != source_highwaters:
print("There is lag. Waiting!")
await asyncio.sleep(2.0)
else:
return
async def check_version(self, node_id=None):
"""Attempt to guess the broker version"""
if node_id is None:
default_group_conns = [
n_id for (n_id, group) in self._conns.keys()
if group == ConnectionGroup.DEFAULT
]
if default_group_conns:
node_id = default_group_conns[0]
else:
assert self.cluster.brokers(), 'no brokers in metadata'
node_id = list(self.cluster.brokers())[0].nodeId
from kafka.protocol.admin import (ListGroupsRequest_v0,
ApiVersionRequest_v0)
from kafka.protocol.commit import (OffsetFetchRequest_v0,
GroupCoordinatorRequest_v0)
from kafka.protocol.metadata import MetadataRequest_v0
test_cases = [
((0, 10), ApiVersionRequest_v0()),
((0, 9), ListGroupsRequest_v0()),
((0, 8, 2), GroupCoordinatorRequest_v0('aiokafka-default-group')),
((0, 8, 1), OffsetFetchRequest_v0('aiokafka-default-group', [])),
((0, 8, 0), MetadataRequest_v0([])),
]
# kafka kills the connection when it doesnt recognize an API request
# so we can send a test request and then follow immediately with a
# vanilla MetadataRequest. If the server did not recognize the first
# request, both will be failed with a ConnectionError that wraps
# socket.error (32, 54, or 104)
conn = await self._get_conn(node_id, no_hint=True)
if conn is None:
raise ConnectionError(
"No connection to node with id {}".format(node_id))
for version, request in test_cases:
try:
if not conn.connected():
await conn.connect()
assert conn, 'no connection to node with id {}'.format(node_id)
# request can be ignored by Kafka broker,
# so we send metadata request and wait response
task = self._loop.create_task(conn.send(request))
await asyncio.wait([task], timeout=0.1, loop=self._loop)
try:
await conn.send(MetadataRequest_v0([]))
except KafkaError:
# metadata request can be cancelled in case
# of invalid correlationIds order
pass
response = await task
except KafkaError:
continue
else:
# To avoid having a connection in undefined state
if node_id != "bootstrap" and conn.connected():
conn.close()
if isinstance(request, ApiVersionRequest_v0):
# Starting from 0.10 kafka broker we determine version
# by looking at ApiVersionResponse
version = self._check_api_version_response(response)
return version
raise UnrecognizedBrokerVersion()
