def _send_commit_request(self, retry_delay=None, attempt=None):
"""Send a commit request with our last_processed_offset"""
# If there's a _commit_call, and it's not active, clear it, it probably
# just called us...
if self._commit_call and not self._commit_call.active():
self._commit_call = None
# Make sure we only have one outstanding commit request at a time
if self._commit_req is not None:
raise OperationInProgress(self._commit_req)
# Handle defaults
if retry_delay is None:
retry_delay = self.retry_init_delay
if attempt is None:
attempt = 1
# Create new OffsetCommitRequest with the latest processed offset
commit_offset = self._last_processed_offset
commit_request = OffsetCommitRequest(
self.topic, self.partition, commit_offset,
TIMESTAMP_INVALID, self.commit_metadata)
log.debug("Committing off=%d grp=%s tpc=%s part=%s req=%r",
self._last_processed_offset, self.consumer_group,
self.topic, self.partition, commit_request)
# Send the request, add our callbacks
self._commit_req = d = self.client.send_offset_commit_request(
self.consumer_group, [commit_request])
d.addBoth(self._clear_commit_req)
d.addCallbacks(
self._update_committed_offset, self._handle_commit_error,
callbackArgs=(commit_offset,),
errbackArgs=(retry_delay, attempt))
def test_commit_fetch_offsets(self):
"""
Commit offsets, then fetch them to verify that the commit succeeded.
"""
# RANT: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
# implies that the metadata supplied with the commit will be returned by
# the fetch, but under 0.8.2.1 with a API_version of 0, it's not. Switch
# to using the V1 API and it works.
c_group = "CG_1"
metadata = "My_Metadata_{}".format(random_string(10)).encode('ascii')
offset = random.randint(0, 1024)
log.debug("Committing offset: %d metadata: %s for topic: %s part: 0",
offset, metadata, self.topic)
req = OffsetCommitRequest(self.topic, 0, offset, -1, metadata)
# We have to retry, since the client doesn't, and Kafka will
# create the topic on the fly, but the first request will fail
[resp] = yield self.retry_while_broker_errors(
self.client.send_offset_commit_request, c_group, [req])
self.assertEqual(getattr(resp, 'error', -1), 0)
req = OffsetFetchRequest(self.topic, 0)
[resp] = yield self.client.send_offset_fetch_request(c_group, [req])
self.assertEqual(resp.error, 0)
self.assertEqual(resp.offset, offset)
# Check we received the proper metadata in the response
self.assertEqual(resp.metadata, metadata)
log.debug("test_commit_fetch_offsets: Test Complete.")
def test_encode_offset_commit_request(self):
header = b"".join([
struct.pack('>h', 8), # Message type = offset commit
struct.pack('>h', 1), # API version
struct.pack('>i', 42), # Correlation ID
struct.pack('>h9s', 9, b"client_id"), # The client ID
struct.pack('>h8s', 8, b"group_id"), # The group to commit for
struct.pack('>i', 996), # Group generation ID
struct.pack('>h11s', 11, b'consumer_id'), # Consumer ID
struct.pack('>i', 2), # Num topics
])
topic1 = b"".join([
struct.pack(">h6s", 6, b"topic1"), # Topic for the request
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 0), # Partition 0
struct.pack(">q", 123), # Offset 123
struct.pack(">q", 1437585816816), # Timestamp in ms > epoch
struct.pack(">h", -1), # Null metadata
struct.pack(">i", 1), # Partition 1
struct.pack(">q", 234), # Offset 234
struct.pack(">q", 1436981054199), # Timestamp in ms > epoch
struct.pack(">h11s", 11, b'My_Metadata'), # Null metadata
])
topic2 = b"".join([
struct.pack(">h6s", 6, b"topic2"), # Topic for the request
struct.pack(">i", 1), # One partition
struct.pack(">i", 2), # Partition 2
struct.pack(">q", 345), # Offset 345
struct.pack(">q", -1), # Timestamp 'invalid-time'
struct.pack(">h", -1), # Null metadata
])
# A dict is used, so we can't predict the order of the topics...
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_offset_commit_request(
b"client_id", 42, b"group_id", 996, b'consumer_id', [
OffsetCommitRequest(b"topic1", 0, 123, 1437585816816, None),
OffsetCommitRequest(b"topic1", 1, 234, 1436981054199,
b'My_Metadata'),
OffsetCommitRequest(b"topic2", 2, 345, -1, None),
])
self.assertIn(encoded, [expected1, expected2])
def test_commit_fetch_offsets(self):
"""test_commit_fetch_offsets
RANT: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
implies that the metadata supplied with the commit will be returned by
the fetch, but under 0.8.2.1 with a API_version of 0, it's not. Switch
to using the V1 API and it works.
""" # noqa
resp = {}
c_group = "CG_1"
metadata = "My_Metadata_{}".format(random_string(10))
offset = random.randint(0, 1024)
log.debug("Commiting offset: %d metadata: %s for topic: %s part: 0",
offset, metadata, self.topic)
req = OffsetCommitRequest(self.topic, 0, offset, -1, metadata)
# We have to retry, since the client doesn't, and Kafka will
# create the topic on the fly, but the first request will fail
for attempt in range(20):
log.debug("test_commit_fetch_offsets: Commit Attempt: %d", attempt)
try:
(resp, ) = yield self.client.send_offset_commit_request(
c_group, [req])
except ConsumerCoordinatorNotAvailableError:
log.info(
"No Coordinator for Consumer Group: %s Attempt: %d of 20",
c_group, attempt)
time.sleep(0.5)
continue
except NotCoordinatorForConsumerError: # pragma: no cover
# Kafka seems to have a timing issue: If we ask broker 'A' who
# the ConsumerCoordinator is for a auto-created, not extant
# topic, the assigned broker may not realize it's been so
# designated by the time we find out and make our request.
log.info(
"Coordinator is not coordinator!!: %s Attempt: %d of 20",
c_group, attempt)
time.sleep(0.5)
continue
break
self.assertEqual(getattr(resp, 'error', -1), 0)
req = OffsetFetchRequest(self.topic, 0)
(resp, ) = yield self.client.send_offset_fetch_request(c_group, [req])
self.assertEqual(resp.error, 0)
self.assertEqual(resp.offset, offset)
# Check we received the proper metadata in the response
self.assertEqual(resp.metadata, metadata)
log.debug("test_commit_fetch_offsets: Test Complete.")
