def _send_commit_request(self, retry_delay=None, attempt=None): """Send a commit request with our last_processed_offset""" # If there's a _commit_call, and it's not active, clear it, it probably # just called us... if self._commit_call and not self._commit_call.active(): self._commit_call = None # Make sure we only have one outstanding commit request at a time if self._commit_req is not None: raise OperationInProgress(self._commit_req) # Handle defaults if retry_delay is None: retry_delay = self.retry_init_delay if attempt is None: attempt = 1 # Create new OffsetCommitRequest with the latest processed offset commit_offset = self._last_processed_offset commit_request = OffsetCommitRequest( self.topic, self.partition, commit_offset, TIMESTAMP_INVALID, self.commit_metadata) log.debug("Committing off=%d grp=%s tpc=%s part=%s req=%r", self._last_processed_offset, self.consumer_group, self.topic, self.partition, commit_request) # Send the request, add our callbacks self._commit_req = d = self.client.send_offset_commit_request( self.consumer_group, [commit_request]) d.addBoth(self._clear_commit_req) d.addCallbacks( self._update_committed_offset, self._handle_commit_error, callbackArgs=(commit_offset,), errbackArgs=(retry_delay, attempt))
def test_commit_fetch_offsets(self): """ Commit offsets, then fetch them to verify that the commit succeeded. """ # RANT: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol # implies that the metadata supplied with the commit will be returned by # the fetch, but under 0.8.2.1 with a API_version of 0, it's not. Switch # to using the V1 API and it works. c_group = "CG_1" metadata = "My_Metadata_{}".format(random_string(10)).encode('ascii') offset = random.randint(0, 1024) log.debug("Committing offset: %d metadata: %s for topic: %s part: 0", offset, metadata, self.topic) req = OffsetCommitRequest(self.topic, 0, offset, -1, metadata) # We have to retry, since the client doesn't, and Kafka will # create the topic on the fly, but the first request will fail [resp] = yield self.retry_while_broker_errors( self.client.send_offset_commit_request, c_group, [req]) self.assertEqual(getattr(resp, 'error', -1), 0) req = OffsetFetchRequest(self.topic, 0) [resp] = yield self.client.send_offset_fetch_request(c_group, [req]) self.assertEqual(resp.error, 0) self.assertEqual(resp.offset, offset) # Check we received the proper metadata in the response self.assertEqual(resp.metadata, metadata) log.debug("test_commit_fetch_offsets: Test Complete.")
def test_encode_offset_commit_request(self): header = b"".join([ struct.pack('>h', 8), # Message type = offset commit struct.pack('>h', 1), # API version struct.pack('>i', 42), # Correlation ID struct.pack('>h9s', 9, b"client_id"), # The client ID struct.pack('>h8s', 8, b"group_id"), # The group to commit for struct.pack('>i', 996), # Group generation ID struct.pack('>h11s', 11, b'consumer_id'), # Consumer ID struct.pack('>i', 2), # Num topics ]) topic1 = b"".join([ struct.pack(">h6s", 6, b"topic1"), # Topic for the request struct.pack(">i", 2), # Two partitions struct.pack(">i", 0), # Partition 0 struct.pack(">q", 123), # Offset 123 struct.pack(">q", 1437585816816), # Timestamp in ms > epoch struct.pack(">h", -1), # Null metadata struct.pack(">i", 1), # Partition 1 struct.pack(">q", 234), # Offset 234 struct.pack(">q", 1436981054199), # Timestamp in ms > epoch struct.pack(">h11s", 11, b'My_Metadata'), # Null metadata ]) topic2 = b"".join([ struct.pack(">h6s", 6, b"topic2"), # Topic for the request struct.pack(">i", 1), # One partition struct.pack(">i", 2), # Partition 2 struct.pack(">q", 345), # Offset 345 struct.pack(">q", -1), # Timestamp 'invalid-time' struct.pack(">h", -1), # Null metadata ]) # A dict is used, so we can't predict the order of the topics... expected1 = b"".join([header, topic1, topic2]) expected2 = b"".join([header, topic2, topic1]) encoded = KafkaCodec.encode_offset_commit_request( b"client_id", 42, b"group_id", 996, b'consumer_id', [ OffsetCommitRequest(b"topic1", 0, 123, 1437585816816, None), OffsetCommitRequest(b"topic1", 1, 234, 1436981054199, b'My_Metadata'), OffsetCommitRequest(b"topic2", 2, 345, -1, None), ]) self.assertIn(encoded, [expected1, expected2])
def test_commit_fetch_offsets(self): """test_commit_fetch_offsets RANT: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol implies that the metadata supplied with the commit will be returned by the fetch, but under 0.8.2.1 with a API_version of 0, it's not. Switch to using the V1 API and it works. """ # noqa resp = {} c_group = "CG_1" metadata = "My_Metadata_{}".format(random_string(10)) offset = random.randint(0, 1024) log.debug("Commiting offset: %d metadata: %s for topic: %s part: 0", offset, metadata, self.topic) req = OffsetCommitRequest(self.topic, 0, offset, -1, metadata) # We have to retry, since the client doesn't, and Kafka will # create the topic on the fly, but the first request will fail for attempt in range(20): log.debug("test_commit_fetch_offsets: Commit Attempt: %d", attempt) try: (resp, ) = yield self.client.send_offset_commit_request( c_group, [req]) except ConsumerCoordinatorNotAvailableError: log.info( "No Coordinator for Consumer Group: %s Attempt: %d of 20", c_group, attempt) time.sleep(0.5) continue except NotCoordinatorForConsumerError: # pragma: no cover # Kafka seems to have a timing issue: If we ask broker 'A' who # the ConsumerCoordinator is for a auto-created, not extant # topic, the assigned broker may not realize it's been so # designated by the time we find out and make our request. log.info( "Coordinator is not coordinator!!: %s Attempt: %d of 20", c_group, attempt) time.sleep(0.5) continue break self.assertEqual(getattr(resp, 'error', -1), 0) req = OffsetFetchRequest(self.topic, 0) (resp, ) = yield self.client.send_offset_fetch_request(c_group, [req]) self.assertEqual(resp.error, 0) self.assertEqual(resp.offset, offset) # Check we received the proper metadata in the response self.assertEqual(resp.metadata, metadata) log.debug("test_commit_fetch_offsets: Test Complete.")