def test_encode_offset_fetch_request(self):
header = b"".join([
struct.pack('>h', 9), # Message type = offset fetch
struct.pack('>h', 1), # API version
struct.pack('>i', 42), # Correlation ID
struct.pack('>h9s', 9, b"client_id"), # The client ID
struct.pack('>h8s', 8, b"group_id"), # The group to commit for
struct.pack('>i', 2), # Num topics
])
topic1 = b"".join([
struct.pack(">h6s", 6, b"topic1"), # Topic for the request
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 0), # Partition 0
struct.pack(">i", 1), # Partition 1
])
topic2 = b"".join([
struct.pack(">h6s", 6, b"topic2"), # Topic for the request
struct.pack(">i", 1), # One partitions
struct.pack(">i", 2), # Partition 2
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_offset_fetch_request(
b"client_id", 42, b"group_id", [
OffsetFetchRequest(b"topic1", 0),
OffsetFetchRequest(b"topic1", 1),
OffsetFetchRequest(b"topic2", 2),
])
self.assertIn(encoded, [expected1, expected2])
def test_commit_fetch_offsets(self):
"""
Commit offsets, then fetch them to verify that the commit succeeded.
"""
# RANT: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
# implies that the metadata supplied with the commit will be returned by
# the fetch, but under 0.8.2.1 with a API_version of 0, it's not. Switch
# to using the V1 API and it works.
c_group = "CG_1"
metadata = "My_Metadata_{}".format(random_string(10)).encode('ascii')
offset = random.randint(0, 1024)
log.debug("Committing offset: %d metadata: %s for topic: %s part: 0",
offset, metadata, self.topic)
req = OffsetCommitRequest(self.topic, 0, offset, -1, metadata)
# We have to retry, since the client doesn't, and Kafka will
# create the topic on the fly, but the first request will fail
[resp] = yield self.retry_while_broker_errors(
self.client.send_offset_commit_request, c_group, [req])
self.assertEqual(getattr(resp, 'error', -1), 0)
req = OffsetFetchRequest(self.topic, 0)
[resp] = yield self.client.send_offset_fetch_request(c_group, [req])
self.assertEqual(resp.error, 0)
self.assertEqual(resp.offset, offset)
# Check we received the proper metadata in the response
self.assertEqual(resp.metadata, metadata)
log.debug("test_commit_fetch_offsets: Test Complete.")
def test_commit_fetch_offsets(self):
"""test_commit_fetch_offsets
RANT: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
implies that the metadata supplied with the commit will be returned by
the fetch, but under 0.8.2.1 with a API_version of 0, it's not. Switch
to using the V1 API and it works.
""" # noqa
resp = {}
c_group = "CG_1"
metadata = "My_Metadata_{}".format(random_string(10))
offset = random.randint(0, 1024)
log.debug("Commiting offset: %d metadata: %s for topic: %s part: 0",
offset, metadata, self.topic)
req = OffsetCommitRequest(self.topic, 0, offset, -1, metadata)
# We have to retry, since the client doesn't, and Kafka will
# create the topic on the fly, but the first request will fail
for attempt in range(20):
log.debug("test_commit_fetch_offsets: Commit Attempt: %d", attempt)
try:
(resp, ) = yield self.client.send_offset_commit_request(
c_group, [req])
except ConsumerCoordinatorNotAvailableError:
log.info(
"No Coordinator for Consumer Group: %s Attempt: %d of 20",
c_group, attempt)
time.sleep(0.5)
continue
except NotCoordinatorForConsumerError: # pragma: no cover
# Kafka seems to have a timing issue: If we ask broker 'A' who
# the ConsumerCoordinator is for a auto-created, not extant
# topic, the assigned broker may not realize it's been so
# designated by the time we find out and make our request.
log.info(
"Coordinator is not coordinator!!: %s Attempt: %d of 20",
c_group, attempt)
time.sleep(0.5)
continue
break
self.assertEqual(getattr(resp, 'error', -1), 0)
req = OffsetFetchRequest(self.topic, 0)
(resp, ) = yield self.client.send_offset_fetch_request(c_group, [req])
self.assertEqual(resp.error, 0)
self.assertEqual(resp.offset, offset)
# Check we received the proper metadata in the response
self.assertEqual(resp.metadata, metadata)
log.debug("test_commit_fetch_offsets: Test Complete.")
def _do_fetch(self):
"""Send a fetch request if there isn't a request outstanding
Sends a fetch request to the Kafka cluster to get messages at the
current offset. When the response comes back, if there are messages,
it delivers them to the :attr:`processor` callback and initiates
another fetch request. If there is a recoverable error, the fetch is
retried after :attr:`retry_delay`.
In the case of an unrecoverable error, :func:`errback` is called on the
:class:`Deferred` returned by :meth:`start()`.
"""
# Check for outstanding request.
if self._request_d:
log.debug("_do_fetch: Outstanding request: %r", self._request_d)
return
# Cleanup our _retry_call, if we have one
if self._retry_call is not None:
if self._retry_call.active():
self._retry_call.cancel()
self._retry_call = None
# Do we know our offset yet, or do we need to figure it out?
if (self._fetch_offset == OFFSET_EARLIEST or
self._fetch_offset == OFFSET_LATEST):
# We need to fetch the offset for our topic/partition
offset_request = OffsetRequest(
self.topic, self.partition, self._fetch_offset, 1)
self._request_d = self.client.send_offset_request([offset_request])
self._request_d.addCallbacks(
self._handle_offset_response, self._handle_offset_error)
elif self._fetch_offset == OFFSET_COMMITTED:
# We need to fetch the committed offset for our topic/partition
# Note we use the same callbacks, as the responses are "close
# enough" for our needs here
if not self.consumer_group:
# consumer_group must be set for OFFSET_COMMITTED
failure = Failure(
InvalidConsumerGroupError("Bad Group_id:{0!r}".format(
self.consumer_group)))
self._start_d.errback(failure)
request = OffsetFetchRequest(self.topic, self.partition)
self._request_d = self.client.send_offset_fetch_request(
self.consumer_group, [request])
self._request_d.addCallbacks(
self._handle_offset_response, self._handle_offset_error)
else:
# Create fetch request payload for our partition
request = FetchRequest(
self.topic, self.partition, self._fetch_offset,
self.buffer_size)
# Send request and add handlers for the response
self._request_d = self.client.send_fetch_request(
[request], max_wait_time=self.fetch_max_wait_time,
min_bytes=self.fetch_min_bytes)
# We need a temp for this because if the response is already
# available, _handle_fetch_response() will clear self._request_d
d = self._request_d
d.addCallback(self._handle_fetch_response)
d.addErrback(self._handle_fetch_error)