def commit(self):
"""
Commit the offset of the message we last processed if it is different
from what we believe is the last offset committed to Kafka.
.. note::
It is possible to commit a smaller offset than Kafka has stored.
This is by design, so we can reprocess a Kafka message stream if
desired.
On error, will retry according to :attr:`request_retry_max_attempts`
(by default, forever).
If called while a commit operation is in progress, and new messages
have been processed since the last request was sent then the commit
will fail with :exc:`OperationInProgress`. The
:exc:`OperationInProgress` exception wraps
a :class:`~twisted.internet.defer.Deferred` which fires when the
outstanding commit operation completes.
:returns:
A :class:`~twisted.internet.defer.Deferred` which resolves with the
committed offset when the operation has completed. It will resolve
immediately if the current offset and the last committed offset do
not differ.
"""
# Can't commit without a consumer_group
if not self.consumer_group:
return fail(Failure(InvalidConsumerGroupError(
"Bad Group_id:{0!r}".format(self.consumer_group))))
# short circuit if we are 'up to date', or haven't processed anything
if ((self._last_processed_offset is None) or
(self._last_processed_offset == self._last_committed_offset)):
return succeed(self._last_committed_offset)
# If we're currently processing a commit we return a failure
# with a deferred we'll fire when the in-progress one completes
if self._commit_ds:
d = Deferred()
self._commit_ds.append(d)
return fail(OperationInProgress(d))
# Ok, we have processed messages since our last commit attempt, and
# we're not currently waiting on a commit request to complete:
# Start a new one
d = Deferred()
self._commit_ds.append(d)
# Send the request
self._send_commit_request()
# Reset the commit_looper here, rather than on success to give
# more stability to the commit interval.
if self._commit_looper is not None:
self._commit_looper.reset()
# return the deferred
return d
def _do_fetch(self):
"""Send a fetch request if there isn't a request outstanding
Sends a fetch request to the Kafka cluster to get messages at the
current offset. When the response comes back, if there are messages,
it delivers them to the :attr:`processor` callback and initiates
another fetch request. If there is a recoverable error, the fetch is
retried after :attr:`retry_delay`.
In the case of an unrecoverable error, :func:`errback` is called on the
:class:`Deferred` returned by :meth:`start()`.
"""
# Check for outstanding request.
if self._request_d:
log.debug("_do_fetch: Outstanding request: %r", self._request_d)
return
# Cleanup our _retry_call, if we have one
if self._retry_call is not None:
if self._retry_call.active():
self._retry_call.cancel()
self._retry_call = None
# Do we know our offset yet, or do we need to figure it out?
if (self._fetch_offset == OFFSET_EARLIEST or
self._fetch_offset == OFFSET_LATEST):
# We need to fetch the offset for our topic/partition
offset_request = OffsetRequest(
self.topic, self.partition, self._fetch_offset, 1)
self._request_d = self.client.send_offset_request([offset_request])
self._request_d.addCallbacks(
self._handle_offset_response, self._handle_offset_error)
elif self._fetch_offset == OFFSET_COMMITTED:
# We need to fetch the committed offset for our topic/partition
# Note we use the same callbacks, as the responses are "close
# enough" for our needs here
if not self.consumer_group:
# consumer_group must be set for OFFSET_COMMITTED
failure = Failure(
InvalidConsumerGroupError("Bad Group_id:{0!r}".format(
self.consumer_group)))
self._start_d.errback(failure)
request = OffsetFetchRequest(self.topic, self.partition)
self._request_d = self.client.send_offset_fetch_request(
self.consumer_group, [request])
self._request_d.addCallbacks(
self._handle_offset_response, self._handle_offset_error)
else:
# Create fetch request payload for our partition
request = FetchRequest(
self.topic, self.partition, self._fetch_offset,
self.buffer_size)
# Send request and add handlers for the response
self._request_d = self.client.send_fetch_request(
[request], max_wait_time=self.fetch_max_wait_time,
min_bytes=self.fetch_min_bytes)
# We need a temp for this because if the response is already
# available, _handle_fetch_response() will clear self._request_d
d = self._request_d
d.addCallback(self._handle_fetch_response)
d.addErrback(self._handle_fetch_error)