def _raise_connection_error(self):
# Cleanup socket if we have one
if self._sock:
self.close()
# And then raise
raise ConnectionError("Kafka @ {0}:{1} went away".format(self.host, self.port))
def _read(self):
try:
while True:
resp = yield from self._reader.readexactly(4)
size, = self.HEADER.unpack(resp)
resp = yield from self._reader.readexactly(size)
fut = self._requests.pop(0)
if not fut.cancelled():
fut.set_result(resp)
except OSError as exc:
conn_exc = ConnectionError("Kafka at {0}:{1} went away".format(
self._host, self._port))
conn_exc.__cause__ = exc
conn_exc.__context__ = exc
fut = self._requests.pop(0)
fut.set_exception(conn_exc)
self.close()
def bootstrap(self):
"""Try to to bootstrap initial cluster metadata"""
# using request v0 for bootstap (bcs api version is not detected yet)
metadata_request = MetadataRequest[0]([])
for host, port, _ in self.hosts:
log.debug("Attempting to bootstrap via node at %s:%s", host, port)
try:
bootstrap_conn = yield from create_conn(
host,
port,
loop=self._loop,
client_id=self._client_id,
request_timeout_ms=self._request_timeout_ms,
ssl_context=self._ssl_context,
security_protocol=self._security_protocol,
max_idle_ms=self._connections_max_idle_ms)
except (OSError, asyncio.TimeoutError) as err:
log.error('Unable connect to "%s:%s": %s', host, port, err)
continue
try:
metadata = yield from bootstrap_conn.send(metadata_request)
except KafkaError as err:
log.warning('Unable to request metadata from "%s:%s": %s',
host, port, err)
bootstrap_conn.close()
continue
self.cluster.update_metadata(metadata)
# A cluster with no topics can return no broker metadata
# in that case, we should keep the bootstrap connection
if not len(self.cluster.brokers()):
self._conns['bootstrap'] = bootstrap_conn
else:
bootstrap_conn.close()
log.debug('Received cluster metadata: %s', self.cluster)
break
else:
raise ConnectionError('Unable to bootstrap from {}'.format(
self.hosts))
# detect api version if need
if self._api_version == 'auto':
self._api_version = yield from self.check_version()
if type(self._api_version) is not tuple:
self._api_version = tuple(map(int, self._api_version.split('.')))
if self._sync_task is None:
# starting metadata synchronizer task
self._sync_task = ensure_future(self._md_synchronizer(),
loop=self._loop)
def check_version(self, node_id=None):
"""Attempt to guess the broker version"""
if node_id is None:
if self._conns:
node_id = list(self._conns.keys())[0]
else:
assert self.cluster.brokers(), 'no brokers in metadata'
node_id = list(self.cluster.brokers())[0].nodeId
from kafka.protocol.admin import (ListGroupsRequest_v0,
ApiVersionRequest_v0)
from kafka.protocol.commit import (OffsetFetchRequest_v0,
GroupCoordinatorRequest_v0)
from kafka.protocol.metadata import MetadataRequest_v0
test_cases = [
('0.10', ApiVersionRequest_v0()),
('0.9', ListGroupsRequest_v0()),
('0.8.2', GroupCoordinatorRequest_v0('aiokafka-default-group')),
('0.8.1', OffsetFetchRequest_v0('aiokafka-default-group', [])),
('0.8.0', MetadataRequest_v0([])),
]
# kafka kills the connection when it doesnt recognize an API request
# so we can send a test request and then follow immediately with a
# vanilla MetadataRequest. If the server did not recognize the first
# request, both will be failed with a ConnectionError that wraps
# socket.error (32, 54, or 104)
conn = yield from self._get_conn(node_id)
if conn is None:
raise ConnectionError(
"No connection to node with id {}".format(node_id))
for version, request in test_cases:
try:
if not conn.connected():
yield from conn.connect()
assert conn, 'no connection to node with id {}'.format(node_id)
# request can be ignored by Kafka broker,
# so we send metadata request and wait response
task = self._loop.create_task(conn.send(request))
yield from asyncio.wait([task], timeout=0.1, loop=self._loop)
try:
yield from conn.send(MetadataRequest_v0([]))
except KafkaError:
# metadata request can be cancelled in case
# of invalid correlationIds order
pass
yield from task
except KafkaError:
continue
else:
return version
raise UnrecognizedBrokerVersion()
def bootstrap(self):
"""Try to to bootstrap initial cluster metadata"""
metadata_request = MetadataRequest([])
for host, port, _ in self.hosts:
log.debug("Attempting to bootstrap via node at %s:%s", host, port)
try:
bootstrap_conn = yield from create_conn(
host,
port,
loop=self._loop,
client_id=self._client_id,
request_timeout_ms=self._request_timeout_ms)
except (OSError, asyncio.TimeoutError) as err:
log.error('Unable connect to "%s:%s": %s', host, port, err)
continue
try:
metadata = yield from bootstrap_conn.send(metadata_request)
except KafkaError as err:
log.warning('Unable to request metadata from "%s:%s": %s',
host, port, err)
bootstrap_conn.close()
continue
self.cluster.update_metadata(metadata)
# A cluster with no topics can return no broker metadata
# in that case, we should keep the bootstrap connection
if not len(self.cluster.brokers()):
self._conns['bootstrap'] = bootstrap_conn
else:
bootstrap_conn.close()
log.debug('Received cluster metadata: %s', self.cluster)
break
else:
raise ConnectionError('Unable to bootstrap from {}'.format(
self.hosts))
if self._sync_task is None:
# starting metadata synchronizer task
self._sync_task = ensure_future(self._md_synchronizer(),
loop=self._loop)
def check_version(self, node_id=None):
"""Attempt to guess the broker version"""
if node_id is None:
if self._conns:
node_id = list(self._conns.keys())[0]
else:
assert self.cluster.brokers(), 'no brokers in metadata'
node_id = list(self.cluster.brokers())[0].nodeId
from kafka.protocol.admin import ListGroupsRequest_v0
from kafka.protocol.commit import (OffsetFetchRequest_v0,
GroupCoordinatorRequest_v0)
from kafka.protocol.metadata import MetadataRequest_v0
test_cases = [
('0.9', ListGroupsRequest_v0()),
('0.8.2', GroupCoordinatorRequest_v0('aiokafka-default-group')),
('0.8.1', OffsetFetchRequest_v0('aiokafka-default-group', [])),
('0.8.0', MetadataRequest_v0([])),
]
# kafka kills the connection when it doesnt recognize an API request
# so we can send a test request and then follow immediately with a
# vanilla MetadataRequest. If the server did not recognize the first
# request, both will be failed with a ConnectionError that wraps
# socket.error (32, 54, or 104)
conn = yield from self._get_conn(node_id)
if conn is None:
raise ConnectionError(
"No connection to node with id {}".format(node_id))
for version, request in test_cases:
try:
if not conn.connected():
yield from conn.connect()
assert conn, 'no connection to node with id {}'.format(node_id)
yield from conn.send(request)
except KafkaError:
continue
else:
return version
raise UnrecognizedBrokerVersion()
def _raise_connection_error(self):
self._dirty = True
raise ConnectionError("Kafka @ {0}:{1} went away".format(
self.host, self.port))