def fetch_last_known_offsets(self, partitions=None):
if self.group is None:
raise ValueError('SimpleClient.group must not be None')
if partitions is None:
partitions = self.client.get_partition_ids_for_topic(self.topic)
responses = self.client.send_offset_fetch_request(
self.group,
[OffsetFetchRequestPayload(self.topic, p) for p in partitions],
fail_on_error=False
)
for resp in responses:
try:
check_error(resp)
# API spec says server wont set an error here
# but 0.8.1.1 does actually...
except UnknownTopicOrPartitionError:
pass
# -1 offset signals no commit is currently stored
if resp.offset == -1:
self.offsets[resp.partition] = 0
# Otherwise we committed the stored offset
# and need to fetch the next one
else:
self.offsets[resp.partition] = resp.offset
def fetch_last_known_offsets(self, partitions=None):
if self.group is None:
raise ValueError('SimpleClient.group must not be None')
if partitions is None:
partitions = self.client.get_partition_ids_for_topic(self.topic)
responses = self.client.send_offset_fetch_request(
self.group,
[OffsetFetchRequestPayload(self.topic, p) for p in partitions],
fail_on_error=False)
for resp in responses:
try:
check_error(resp)
# API spec says server wont set an error here
# but 0.8.1.1 does actually...
except UnknownTopicOrPartitionError:
pass
# -1 offset signals no commit is currently stored
if resp.offset == -1:
self.offsets[resp.partition] = 0
# Otherwise we committed the stored offset
# and need to fetch the next one
else:
self.offsets[resp.partition] = resp.offset
def _check_commit_response_error(resp):
try:
check_error(resp)
except BrokerResponseError as e:
exception = OffsetCommitError(resp.topic, resp.partition, e.message)
return exception
return resp
def _check_commit_response_error(resp):
try:
check_error(resp)
except BrokerResponseError as e:
exception = OffsetCommitError(
resp.topic,
resp.partition,
e.message
)
return exception
return resp
def _check_fetch_response_error(resp):
try:
check_error(resp)
except BrokerResponseError:
# In case of error we set the offset to (-1,)
return OffsetResponsePayload(
resp.topic,
resp.partition,
resp.error,
(-1,),
)
return resp
def commit_partition_offsets(self, partition_offsets):
"""
Commit explicit partition/offset pairs.
"""
self.logger.debug("Committing partition offsets: %s", partition_offsets)
commit_requests = [
OffsetCommitRequestPayload(self.consumer.topic, partition, offset, None)
for partition, offset in partition_offsets.items()
]
commit_responses = self.consumer.client.send_offset_commit_request(self.consumer.group, commit_requests)
for commit_response in commit_responses:
check_error(commit_response)
def _update_group_offsets(self):
logger.info("Consumer fetching stored offsets")
for partition in self._client.get_partition_ids_for_topic(self._topic):
(resp, ) = self._client.send_offset_fetch_request(
self._group_id, [OffsetFetchRequest(self._topic, partition)],
fail_on_error=False)
try:
check_error(resp)
except UnknownTopicOrPartitionError:
pass
if resp.offset == -1:
self._offsets.commit[partition] = None
else:
self._offsets.commit[partition] = resp.offset
def commit_partition_offsets(self, partition_offsets):
"""
Commit explicit partition/offset pairs.
"""
self.logger.debug("Committing partition offsets: %s", partition_offsets)
commit_requests = [
OffsetCommitRequestPayload(self.consumer.topic, partition, offset, None)
for partition, offset in partition_offsets.items()
]
commit_responses = self.consumer.client.send_offset_commit_request(
self.consumer.group,
commit_requests,
)
for commit_response in commit_responses:
check_error(commit_response)
def pluck_topic_offset_or_zero_on_unknown(resp):
try:
check_error(resp)
except UnknownTopicOrPartitionError:
# If the server doesn't have any commited offsets by this group for
# this topic, assume it's zero.
pass
# The API spec says server wont set an error, but 0.8.1.1 does. The actual
# check is if the offset is -1.
if resp.offset == -1:
return OffsetFetchResponsePayload(
resp.topic,
resp.partition,
0,
resp.metadata,
0,
)
return resp
def _update_produced_offsets(self):
"""
Arguments:
request_time_ms (int): Used to ask for all messages before a
certain time (ms). There are two special values. Specify -1 to receive the latest
offset (i.e. the offset of the next coming message) and -2 to receive the earliest
available offset. Note that because offsets are pulled in descending order, asking for
the earliest offset will always return you a single element.
"""
for partition in self._client.get_partition_ids_for_topic(self._topic):
reqs = [OffsetRequest(self._topic, partition, -1, 1)]
(resp, ) = self._client.send_offset_request(reqs)
check_error(resp)
assert resp.topic == self._topic
assert resp.partition == partition
self._offsets.produced[partition] = resp.offsets[0]
def _fetch(self):
# Create fetch request payloads for all the partitions
partitions = dict((p, self.buffer_size)
for p in self.fetch_offsets.keys())
while partitions:
requests = []
for partition, buffer_size in six.iteritems(partitions):
requests.append(FetchRequestPayload(self.topic, partition,
self.fetch_offsets[partition],
buffer_size))
# Send request
responses = self.client.send_fetch_request(
requests,
max_wait_time=int(self.fetch_max_wait_time),
min_bytes=self.fetch_min_bytes,
fail_on_error=False
)
retry_partitions = {}
for resp in responses:
try:
check_error(resp)
except UnknownTopicOrPartitionError:
log.error('UnknownTopicOrPartitionError for %s:%d',
resp.topic, resp.partition)
self.client.reset_topic_metadata(resp.topic)
raise
except NotLeaderForPartitionError:
log.error('NotLeaderForPartitionError for %s:%d',
resp.topic, resp.partition)
self.client.reset_topic_metadata(resp.topic)
continue
except OffsetOutOfRangeError:
log.warning('OffsetOutOfRangeError for %s:%d. '
'Resetting partition offset...',
resp.topic, resp.partition)
self.reset_partition_offset(resp.partition)
# Retry this partition
retry_partitions[resp.partition] = partitions[resp.partition]
continue
except FailedPayloadsError as e:
log.warning('FailedPayloadsError for %s:%d',
e.payload.topic, e.payload.partition)
# Retry this partition
retry_partitions[e.payload.partition] = partitions[e.payload.partition]
continue
partition = resp.partition
buffer_size = partitions[partition]
# Check for partial message
if resp.messages and isinstance(resp.messages[-1].message, PartialMessage):
# If buffer is at max and all we got was a partial message
# raise ConsumerFetchSizeTooSmall
if (self.max_buffer_size is not None and
buffer_size == self.max_buffer_size and
len(resp.messages) == 1):
log.error('Max fetch size %d too small', self.max_buffer_size)
raise ConsumerFetchSizeTooSmall()
if self.max_buffer_size is None:
buffer_size *= 2
else:
buffer_size = min(buffer_size * 2, self.max_buffer_size)
log.warning('Fetch size too small, increase to %d (2x) '
'and retry', buffer_size)
retry_partitions[partition] = buffer_size
resp.messages.pop()
for message in resp.messages:
if message.offset < self.fetch_offsets[partition]:
log.debug('Skipping message %s because its offset is less than the consumer offset',
message)
continue
# Put the message in our queue
self.queue.put((partition, message))
self.fetch_offsets[partition] = message.offset + 1
partitions = retry_partitions
