DEFAULT_CONFIG = {

'key_deserializer': None,

'value_deserializer': None,

'fetch_min_bytes': 1,

'fetch_max_wait_ms': 500,

'fetch_max_bytes': 52428800,

'max_partition_fetch_bytes': 1048576,

'max_poll_records': sys.maxsize,

'check_crcs': True,

'skip_double_compressed_messages': False,

'iterator_refetch_records': 1, # undocumented -- interface may change

'metric_group_prefix': 'consumer',

'api_version': (0, 8, 0),

'retry_backoff_ms': 100

}

def __init__(self, client, subscriptions, metrics, **configs):

"""Initialize a Kafka Message Fetcher.

Keyword Arguments:

key_deserializer (callable): Any callable that takes a

raw message key and returns a deserialized key.

value_deserializer (callable, optional): Any callable that takes a

raw message value and returns a deserialized value.

fetch_min_bytes (int): Minimum amount of data the server should

return for a fetch request, otherwise wait up to

fetch_max_wait_ms for more data to accumulate. Default: 1.

fetch_max_wait_ms (int): The maximum amount of time in milliseconds

the server will block before answering the fetch request if

there isn't sufficient data to immediately satisfy the

requirement given by fetch_min_bytes. Default: 500.

fetch_max_bytes (int): The maximum amount of data the server should

return for a fetch request. This is not an absolute maximum, if

the first message in the first non-empty partition of the fetch

is larger than this value, the message will still be returned

to ensure that the consumer can make progress. NOTE: consumer

performs fetches to multiple brokers in parallel so memory

usage will depend on the number of brokers containing

partitions for the topic.

Supported Kafka version >= 0.10.1.0. Default: 52428800 (50 MB).

max_partition_fetch_bytes (int): The maximum amount of data

per-partition the server will return. The maximum total memory

used for a request = #partitions * max_partition_fetch_bytes.

This size must be at least as large as the maximum message size

the server allows or else it is possible for the producer to

send messages larger than the consumer can fetch. If that

happens, the consumer can get stuck trying to fetch a large

message on a certain partition. Default: 1048576.

check_crcs (bool): Automatically check the CRC32 of the records

consumed. This ensures no on-the-wire or on-disk corruption to

the messages occurred. This check adds some overhead, so it may

be disabled in cases seeking extreme performance. Default: True

skip_double_compressed_messages (bool): A bug in KafkaProducer

caused some messages to be corrupted via double-compression.

By default, the fetcher will return the messages as a compressed

blob of bytes with a single offset, i.e. how the message was

actually published to the cluster. If you prefer to have the

fetcher automatically detect corrupt messages and skip them,

set this option to True. Default: False.

"""

self.config = copy.copy(self.DEFAULT_CONFIG)

for key in self.config:

if key in configs:

self.config[key] = configs[key]

self._client = client

self._subscriptions = subscriptions

self._completed_fetches = collections.deque() # Unparsed responses

self._next_partition_records = None # Holds a single PartitionRecords until fully consumed

self._iterator = None

self._fetch_futures = collections.deque()

self._sensors = FetchManagerMetrics(metrics, self.config['metric_group_prefix'])

self._isolation_level = READ_UNCOMMITTED

def send_fetches(self):

"""Send FetchRequests for all assigned partitions that do not already have

an in-flight fetch or pending fetch data.

Returns:

List of Futures: each future resolves to a FetchResponse

"""

futures = []

for node_id, request in six.iteritems(self._create_fetch_requests()):

if self._client.ready(node_id):

log.debug("Sending FetchRequest to node %s", node_id)

future = self._client.send(node_id, request)

future.add_callback(self._handle_fetch_response, request, time.time())

future.add_errback(log.error, 'Fetch to node %s failed: %s', node_id)

futures.append(future)

self._fetch_futures.extend(futures)

self._clean_done_fetch_futures()

return futures

def reset_offsets_if_needed(self, partitions):

"""Lookup and set offsets for any partitions which are awaiting an

explicit reset.

Arguments:

partitions (set of TopicPartitions): the partitions to reset

"""

for tp in partitions:

# TODO: If there are several offsets to reset, we could submit offset requests in parallel

if self._subscriptions.is_assigned(tp) and self._subscriptions.is_offset_reset_needed(tp):

self._reset_offset(tp)

def _clean_done_fetch_futures(self):

while True:

if not self._fetch_futures:

break

if not self._fetch_futures[0].is_done:

break

self._fetch_futures.popleft()

def in_flight_fetches(self):

"""Return True if there are any unprocessed FetchRequests in flight."""

self._clean_done_fetch_futures()

return bool(self._fetch_futures)

def update_fetch_positions(self, partitions):

"""Update the fetch positions for the provided partitions.

Arguments:

partitions (list of TopicPartitions): partitions to update

Raises:

NoOffsetForPartitionError: if no offset is stored for a given

partition and no reset policy is available

"""

# reset the fetch position to the committed position

for tp in partitions:

if not self._subscriptions.is_assigned(tp):

log.warning("partition %s is not assigned - skipping offset"

" update", tp)

continue

elif self._subscriptions.is_fetchable(tp):

log.warning("partition %s is still fetchable -- skipping offset"

" update", tp)

continue

if self._subscriptions.is_offset_reset_needed(tp):

self._reset_offset(tp)

elif self._subscriptions.assignment[tp].committed is None:

# there's no committed position, so we need to reset with the

# default strategy

self._subscriptions.need_offset_reset(tp)

self._reset_offset(tp)

else:

committed = self._subscriptions.assignment[tp].committed

log.debug("Resetting offset for partition %s to the committed"

" offset %s", tp, committed)

self._subscriptions.seek(tp, committed)

def get_offsets_by_times(self, timestamps, timeout_ms):

offsets = self._retrieve_offsets(timestamps, timeout_ms)

for tp in timestamps:

if tp not in offsets:

offsets[tp] = None

else:

offset, timestamp = offsets[tp]

offsets[tp] = OffsetAndTimestamp(offset, timestamp)

return offsets

def beginning_offsets(self, partitions, timeout_ms):

return self.beginning_or_end_offset(

partitions, OffsetResetStrategy.EARLIEST, timeout_ms)

def end_offsets(self, partitions, timeout_ms):

return self.beginning_or_end_offset(

partitions, OffsetResetStrategy.LATEST, timeout_ms)

def beginning_or_end_offset(self, partitions, timestamp, timeout_ms):

timestamps = dict([(tp, timestamp) for tp in partitions])

offsets = self._retrieve_offsets(timestamps, timeout_ms)

for tp in timestamps:

offsets[tp] = offsets[tp][0]

return offsets

def _reset_offset(self, partition):

"""Reset offsets for the given partition using the offset reset strategy.

Arguments:

partition (TopicPartition): the partition that needs reset offset

Raises:

NoOffsetForPartitionError: if no offset reset strategy is defined

"""

timestamp = self._subscriptions.assignment[partition].reset_strategy

if timestamp is OffsetResetStrategy.EARLIEST:

strategy = 'earliest'

elif timestamp is OffsetResetStrategy.LATEST:

strategy = 'latest'

else:

raise NoOffsetForPartitionError(partition)

log.debug("Resetting offset for partition %s to %s offset.",

partition, strategy)

offsets = self._retrieve_offsets({partition: timestamp})

if partition not in offsets:

raise NoOffsetForPartitionError(partition)

offset = offsets[partition][0]

# we might lose the assignment while fetching the offset,

# so check it is still active

if self._subscriptions.is_assigned(partition):

self._subscriptions.seek(partition, offset)

def _retrieve_offsets(self, timestamps, timeout_ms=float("inf")):

"""Fetch offset for each partition passed in ``timestamps`` map.

Blocks until offsets are obtained, a non-retriable exception is raised

or ``timeout_ms`` passed.

Arguments:

timestamps: {TopicPartition: int} dict with timestamps to fetch

offsets by. -1 for the latest available, -2 for the earliest

available. Otherwise timestamp is treated as epoch miliseconds.

Returns:

{TopicPartition: (int, int)}: Mapping of partition to

retrieved offset and timestamp. If offset does not exist for

the provided timestamp, that partition will be missing from

this mapping.

"""

if not timestamps:

return {}

start_time = time.time()

remaining_ms = timeout_ms

while remaining_ms > 0:

future = self._send_offset_requests(timestamps)

self._client.poll(future=future, timeout_ms=remaining_ms)

if future.succeeded():

return future.value

if not future.retriable():

raise future.exception # pylint: disable-msg=raising-bad-type

elapsed_ms = (time.time() - start_time) * 1000

remaining_ms = timeout_ms - elapsed_ms

if remaining_ms < 0:

break

if future.exception.invalid_metadata:

refresh_future = self._client.cluster.request_update()

self._client.poll(future=refresh_future, timeout_ms=remaining_ms)

else:

time.sleep(self.config['retry_backoff_ms'] / 1000.0)

elapsed_ms = (time.time() - start_time) * 1000

remaining_ms = timeout_ms - elapsed_ms

raise Errors.KafkaTimeoutError(

"Failed to get offsets by timestamps in %s ms" % timeout_ms)

def fetched_records(self, max_records=None):

"""Returns previously fetched records and updates consumed offsets.

Arguments:

max_records (int): Maximum number of records returned. Defaults

to max_poll_records configuration.

Raises:

OffsetOutOfRangeError: if no subscription offset_reset_strategy

CorruptRecordException: if message crc validation fails (check_crcs

must be set to True)

RecordTooLargeError: if a message is larger than the currently

configured max_partition_fetch_bytes

TopicAuthorizationError: if consumer is not authorized to fetch

messages from the topic

Returns: (records (dict), partial (bool))

records: {TopicPartition: [messages]}

partial: True if records returned did not fully drain any pending

partition requests. This may be useful for choosing when to

pipeline additional fetch requests.

"""

if max_records is None:

max_records = self.config['max_poll_records']

assert max_records > 0

if self._subscriptions.needs_partition_assignment:

return {}, False

drained = collections.defaultdict(list)

records_remaining = max_records

while records_remaining > 0:

if not self._next_partition_records:

if not self._completed_fetches:

break

completion = self._completed_fetches.popleft()

self._next_partition_records = self._parse_fetched_data(completion)

else:

records_remaining -= self._append(drained,

self._next_partition_records,

records_remaining)

return dict(drained), bool(self._completed_fetches)

def _append(self, drained, part, max_records):

if not part:

return 0

tp = part.topic_partition

fetch_offset = part.fetch_offset

if not self._subscriptions.is_assigned(tp):

# this can happen when a rebalance happened before

# fetched records are returned to the consumer's poll call

log.debug("Not returning fetched records for partition %s"

" since it is no longer assigned", tp)

else:

# note that the position should always be available

# as long as the partition is still assigned

position = self._subscriptions.assignment[tp].position

if not self._subscriptions.is_fetchable(tp):

# this can happen when a partition is paused before

# fetched records are returned to the consumer's poll call

log.debug("Not returning fetched records for assigned partition"

" %s since it is no longer fetchable", tp)

elif fetch_offset == position:

# we are ensured to have at least one record since we already checked for emptiness

part_records = part.take(max_records)

next_offset = part_records[-1].offset + 1

log.log(0, "Returning fetched records at offset %d for assigned"

" partition %s and update position to %s", position,

tp, next_offset)

for record in part_records:

# Fetched compressed messages may include additional records

if record.offset < fetch_offset:

log.debug("Skipping message offset: %s (expecting %s)",

record.offset, fetch_offset)

continue

drained[tp].append(record)

self._subscriptions.assignment[tp].position = next_offset

return len(part_records)

else:

# these records aren't next in line based on the last consumed

# position, ignore them they must be from an obsolete request

log.debug("Ignoring fetched records for %s at offset %s since"

" the current position is %d", tp, part.fetch_offset,

position)

part.discard()

return 0

def _message_generator(self):

"""Iterate over fetched_records"""

if self._subscriptions.needs_partition_assignment:

raise StopIteration('Subscription needs partition assignment')

while self._next_partition_records or self._completed_fetches:

if not self._next_partition_records:

completion = self._completed_fetches.popleft()

self._next_partition_records = self._parse_fetched_data(completion)

continue

# Send additional FetchRequests when the internal queue is low

# this should enable moderate pipelining

if len(self._completed_fetches) <= self.config['iterator_refetch_records']:

self.send_fetches()

tp = self._next_partition_records.topic_partition

# We can ignore any prior signal to drop pending message sets

# because we are starting from a fresh one where fetch_offset == position

# i.e., the user seek()'d to this position

self._subscriptions.assignment[tp].drop_pending_message_set = False

for msg in self._next_partition_records.take():

# Because we are in a generator, it is possible for

# subscription state to change between yield calls

# so we need to re-check on each loop

# this should catch assignment changes, pauses

# and resets via seek_to_beginning / seek_to_end

if not self._subscriptions.is_fetchable(tp):

log.debug("Not returning fetched records for partition %s"

" since it is no longer fetchable", tp)

self._next_partition_records = None

break

# If there is a seek during message iteration,

# we should stop unpacking this message set and

# wait for a new fetch response that aligns with the

# new seek position

elif self._subscriptions.assignment[tp].drop_pending_message_set:

log.debug("Skipping remainder of message set for partition %s", tp)

self._subscriptions.assignment[tp].drop_pending_message_set = False

self._next_partition_records = None

break

# Compressed messagesets may include earlier messages

elif msg.offset < self._subscriptions.assignment[tp].position:

log.debug("Skipping message offset: %s (expecting %s)",

msg.offset,

self._subscriptions.assignment[tp].position)

continue

self._subscriptions.assignment[tp].position = msg.offset + 1

yield msg

self._next_partition_records = None

def _unpack_message_set(self, tp, records):

try:

batch = records.next_batch()

while batch is not None:

for record in batch:

key_size = len(record.key) if record.key is not None else -1

value_size = len(record.value) if record.value is not None else -1

key = self._deserialize(

self.config['key_deserializer'],

tp.topic, record.key)

value = self._deserialize(

self.config['value_deserializer'],

tp.topic, record.value)

yield ConsumerRecord(

tp.topic, tp.partition, record.offset, record.timestamp,

record.timestamp_type, key, value, record.checksum,

key_size, value_size)

batch = records.next_batch()

# If unpacking raises StopIteration, it is erroneously

# caught by the generator. We want all exceptions to be raised

# back to the user. See Issue 545

except StopIteration as e:

log.exception('StopIteration raised unpacking messageset: %s', e)

raise Exception('StopIteration raised unpacking messageset')

def __iter__(self): # pylint: disable=non-iterator-returned

return self

def __next__(self):

if not self._iterator:

self._iterator = self._message_generator()

try:

return next(self._iterator)

except StopIteration:

self._iterator = None

raise

def _deserialize(self, f, topic, bytes_):

if not f:

return bytes_

if isinstance(f, Deserializer):

return f.deserialize(topic, bytes_)

return f(bytes_)

def _send_offset_requests(self, timestamps):

"""Fetch offsets for each partition in timestamps dict. This may send

request to multiple nodes, based on who is Leader for partition.

Arguments:

timestamps (dict): {TopicPartition: int} mapping of fetching

timestamps.

Returns:

Future: resolves to a mapping of retrieved offsets

"""

timestamps_by_node = collections.defaultdict(dict)

for partition, timestamp in six.iteritems(timestamps):

node_id = self._client.cluster.leader_for_partition(partition)

if node_id is None:

self._client.add_topic(partition.topic)

log.debug("Partition %s is unknown for fetching offset,"

" wait for metadata refresh", partition)

return Future().failure(Errors.StaleMetadata(partition))

elif node_id == -1:

log.debug("Leader for partition %s unavailable for fetching "

"offset, wait for metadata refresh", partition)

return Future().failure(

Errors.LeaderNotAvailableError(partition))

else:

timestamps_by_node[node_id][partition] = timestamp

# Aggregate results until we have all

list_offsets_future = Future()

responses = []

node_count = len(timestamps_by_node)

def on_success(value):

responses.append(value)

if len(responses) == node_count:

offsets = {}

for r in responses:

offsets.update(r)

list_offsets_future.success(offsets)

def on_fail(err):

if not list_offsets_future.is_done:

list_offsets_future.failure(err)

for node_id, timestamps in six.iteritems(timestamps_by_node):

_f = self._send_offset_request(node_id, timestamps)

_f.add_callback(on_success)

_f.add_errback(on_fail)

return list_offsets_future

def _send_offset_request(self, node_id, timestamps):

by_topic = collections.defaultdict(list)

for tp, timestamp in six.iteritems(timestamps):

if self.config['api_version'] >= (0, 10, 1):

data = (tp.partition, timestamp)

else:

data = (tp.partition, timestamp, 1)

by_topic[tp.topic].append(data)

if self.config['api_version'] >= (0, 10, 1):

request = OffsetRequest[1](-1, list(six.iteritems(by_topic)))

else:

request = OffsetRequest[0](-1, list(six.iteritems(by_topic)))

# Client returns a future that only fails on network issues

# so create a separate future and attach a callback to update it

# based on response error codes

future = Future()

_f = self._client.send(node_id, request)

_f.add_callback(self._handle_offset_response, future)

_f.add_errback(lambda e: future.failure(e))

return future

def _handle_offset_response(self, future, response):

"""Callback for the response of the list offset call above.

Arguments:

future (Future): the future to update based on response

response (OffsetResponse): response from the server

Raises:

AssertionError: if response does not match partition

"""

timestamp_offset_map = {}

for topic, part_data in response.topics:

for partition_info in part_data:

partition, error_code = partition_info[:2]

partition = TopicPartition(topic, partition)

error_type = Errors.for_code(error_code)

if error_type is Errors.NoError:

if response.API_VERSION == 0:

offsets = partition_info[2]

assert len(offsets) <= 1, 'Expected OffsetResponse with one offset'

if not offsets:

offset = UNKNOWN_OFFSET

else:

offset = offsets[0]

log.debug("Handling v0 ListOffsetResponse response for %s. "

"Fetched offset %s", partition, offset)

if offset != UNKNOWN_OFFSET:

timestamp_offset_map[partition] = (offset, None)

else:

timestamp, offset = partition_info[2:]

log.debug("Handling ListOffsetResponse response for %s. "

"Fetched offset %s, timestamp %s",

partition, offset, timestamp)

if offset != UNKNOWN_OFFSET:

timestamp_offset_map[partition] = (offset, timestamp)

elif error_type is Errors.UnsupportedForMessageFormatError:

# The message format on the broker side is before 0.10.0,

# we simply put None in the response.

log.debug("Cannot search by timestamp for partition %s because the"

" message format version is before 0.10.0", partition)

elif error_type is Errors.NotLeaderForPartitionError:

log.debug("Attempt to fetch offsets for partition %s failed due"

" to obsolete leadership information, retrying.",

partition)

future.failure(error_type(partition))

return

elif error_type is Errors.UnknownTopicOrPartitionError:

log.warn("Received unknown topic or partition error in ListOffset "

"request for partition %s. The topic/partition " +

"may not exist or the user may not have Describe access "

"to it.", partition)

future.failure(error_type(partition))

return

else:

log.warning("Attempt to fetch offsets for partition %s failed due to:"

" %s", partition, error_type)

future.failure(error_type(partition))

return

if not future.is_done:

future.success(timestamp_offset_map)

def _fetchable_partitions(self):

fetchable = self._subscriptions.fetchable_partitions()

if self._next_partition_records:

fetchable.remove(self._next_partition_records.topic_partition)

for fetch in self._completed_fetches:

fetchable.remove(fetch.topic_partition)

return fetchable

def _create_fetch_requests(self):

"""Create fetch requests for all assigned partitions, grouped by node.

FetchRequests skipped if no leader, or node has requests in flight

Returns:

dict: {node_id: FetchRequest, ...} (version depends on api_version)

"""

# create the fetch info as a dict of lists of partition info tuples

# which can be passed to FetchRequest() via .items()

fetchable = collections.defaultdict(lambda: collections.defaultdict(list))

for partition in self._fetchable_partitions():

node_id = self._client.cluster.leader_for_partition(partition)

position = self._subscriptions.assignment[partition].position

# fetch if there is a leader and no in-flight requests

if node_id is None or node_id == -1:

log.debug("No leader found for partition %s."

" Requesting metadata update", partition)

self._client.cluster.request_update()

elif self._client.in_flight_request_count(node_id) == 0:

partition_info = (

partition.partition,

position,

self.config['max_partition_fetch_bytes']

)

fetchable[node_id][partition.topic].append(partition_info)

log.debug("Adding fetch request for partition %s at offset %d",

partition, position)

if self.config['api_version'] >= (0, 11, 0):

version = 4

elif self.config['api_version'] >= (0, 10, 1):

version = 3

elif self.config['api_version'] >= (0, 10):

version = 2

elif self.config['api_version'] == (0, 9):

version = 1

else:

version = 0

requests = {}

for node_id, partition_data in six.iteritems(fetchable):

if version < 3:

requests[node_id] = FetchRequest[version](

-1, # replica_id

self.config['fetch_max_wait_ms'],

self.config['fetch_min_bytes'],

partition_data.items())

else:

# As of version == 3 partitions will be returned in order as

# they are requested, so to avoid starvation with

# `fetch_max_bytes` option we need this shuffle

# NOTE: we do have partition_data in random order due to usage

# of unordered structures like dicts, but that does not

# guarantee equal distribution, and starting in Python3.6

# dicts retain insert order.

partition_data = list(partition_data.items())

random.shuffle(partition_data)

if version == 3:

requests[node_id] = FetchRequest[version](

-1, # replica_id

self.config['fetch_max_wait_ms'],

self.config['fetch_min_bytes'],

self.config['fetch_max_bytes'],

partition_data)

else:

requests[node_id] = FetchRequest[version](

-1, # replica_id

self.config['fetch_max_wait_ms'],

self.config['fetch_min_bytes'],

self.config['fetch_max_bytes'],

self._isolation_level,

partition_data)

return requests

def _handle_fetch_response(self, request, send_time, response):

"""The callback for fetch completion"""

fetch_offsets = {}

for topic, partitions in request.topics:

for partition_data in partitions:

partition, offset = partition_data[:2]

fetch_offsets[TopicPartition(topic, partition)] = offset

partitions = set([TopicPartition(topic, partition_data[0])

for topic, partitions in response.topics

for partition_data in partitions])

metric_aggregator = FetchResponseMetricAggregator(self._sensors, partitions)

# randomized ordering should improve balance for short-lived consumers

random.shuffle(response.topics)

for topic, partitions in response.topics:

random.shuffle(partitions)

for partition_data in partitions:

tp = TopicPartition(topic, partition_data[0])

completed_fetch = CompletedFetch(

tp, fetch_offsets[tp],

response.API_VERSION,

partition_data[1:],

metric_aggregator

)

self._completed_fetches.append(completed_fetch)

if response.API_VERSION >= 1:

self._sensors.fetch_throttle_time_sensor.record(response.throttle_time_ms)

self._sensors.fetch_latency.record((time.time() - send_time) * 1000)

def _parse_fetched_data(self, completed_fetch):

tp = completed_fetch.topic_partition

fetch_offset = completed_fetch.fetched_offset

num_bytes = 0

records_count = 0

parsed_records = None

error_code, highwater = completed_fetch.partition_data[:2]

error_type = Errors.for_code(error_code)

try:

if not self._subscriptions.is_fetchable(tp):

# this can happen when a rebalance happened or a partition

# consumption paused while fetch is still in-flight

log.debug("Ignoring fetched records for partition %s"

" since it is no longer fetchable", tp)

elif error_type is Errors.NoError:

self._subscriptions.assignment[tp].highwater = highwater

# we are interested in this fetch only if the beginning

# offset (of the *request*) matches the current consumed position

# Note that the *response* may return a messageset that starts

# earlier (e.g., compressed messages) or later (e.g., compacted topic)

position = self._subscriptions.assignment[tp].position

if position is None or position != fetch_offset:

log.debug("Discarding fetch response for partition %s"

" since its offset %d does not match the"

" expected offset %d", tp, fetch_offset,

position)

return None

records = MemoryRecords(completed_fetch.partition_data[-1])

if records.has_next():

log.debug("Adding fetched record for partition %s with"

" offset %d to buffered record list", tp,

position)

unpacked = list(self._unpack_message_set(tp, records))

parsed_records = self.PartitionRecords(fetch_offset, tp, unpacked)

last_offset = unpacked[-1].offset

self._sensors.records_fetch_lag.record(highwater - last_offset)

num_bytes = records.valid_bytes()

records_count = len(unpacked)

elif records.size_in_bytes() > 0:

# we did not read a single message from a non-empty

# buffer because that message's size is larger than

# fetch size, in this case record this exception

record_too_large_partitions = {tp: fetch_offset}

raise RecordTooLargeError(

"There are some messages at [Partition=Offset]: %s "

" whose size is larger than the fetch size %s"

" and hence cannot be ever returned."

" Increase the fetch size, or decrease the maximum message"

" size the broker will allow." % (

record_too_large_partitions,

self.config['max_partition_fetch_bytes']),

record_too_large_partitions)

self._sensors.record_topic_fetch_metrics(tp.topic, num_bytes, records_count)

elif error_type in (Errors.NotLeaderForPartitionError,

Errors.UnknownTopicOrPartitionError):

self._client.cluster.request_update()

elif error_type is Errors.OffsetOutOfRangeError:

position = self._subscriptions.assignment[tp].position

if position is None or position != fetch_offset:

log.debug("Discarding stale fetch response for partition %s"

" since the fetched offset %d does not match the"

" current offset %d", tp, fetch_offset, position)

elif self._subscriptions.has_default_offset_reset_policy():

log.info("Fetch offset %s is out of range for topic-partition %s", fetch_offset, tp)

self._subscriptions.need_offset_reset(tp)

else:

raise Errors.OffsetOutOfRangeError({tp: fetch_offset})

elif error_type is Errors.TopicAuthorizationFailedError:

log.warn("Not authorized to read from topic %s.", tp.topic)

raise Errors.TopicAuthorizationFailedError(set(tp.topic))

elif error_type is Errors.UnknownError:

log.warn("Unknown error fetching data for topic-partition %s", tp)

else:

raise error_type('Unexpected error while fetching data')

finally:

completed_fetch.metric_aggregator.record(tp, num_bytes, records_count)

return parsed_records

class PartitionRecords(six.Iterator):

def __init__(self, fetch_offset, tp, messages):

self.fetch_offset = fetch_offset

self.topic_partition = tp

self.messages = messages

self.message_idx = 0

# For truthiness evaluation we need to define __len__ or __nonzero__

def __len__(self):

if self.messages is None or self.message_idx >= len(self.messages):

return 0

return len(self.messages) - self.message_idx

def discard(self):

self.messages = None

def take(self, n=None):

if not len(self):

return []

if n is None or n > len(self):

n = len(self)

next_idx = self.message_idx + n

res = self.messages[self.message_idx:next_idx]

self.message_idx = next_idx

if len(self) > 0:

self.fetch_offset = self.messages[self.message_idx].offset

"""

Since we parse the message data for each partition from each fetch

response lazily, fetch-level metrics need to be aggregated as the messages

from each partition are parsed. This class is used to facilitate this

incremental aggregation.

"""

def __init__(self, sensors, partitions):

self.sensors = sensors

self.unrecorded_partitions = partitions

self.total_bytes = 0

self.total_records = 0

def record(self, partition, num_bytes, num_records):

"""

After each partition is parsed, we update the current metric totals

with the total bytes and number of records parsed. After all partitions

have reported, we write the metric.

"""

self.unrecorded_partitions.remove(partition)

self.total_bytes += num_bytes

self.total_records += num_records

# once all expected partitions from the fetch have reported in, record the metrics

if not self.unrecorded_partitions:

self.sensors.bytes_fetched.record(self.total_bytes)

def __init__(self, metrics, prefix):

self.metrics = metrics

self.group_name = '%s-fetch-manager-metrics' % prefix

self.bytes_fetched = metrics.sensor('bytes-fetched')

self.bytes_fetched.add(metrics.metric_name('fetch-size-avg', self.group_name,

'The average number of bytes fetched per request'), Avg())

self.bytes_fetched.add(metrics.metric_name('fetch-size-max', self.group_name,

'The maximum number of bytes fetched per request'), Max())

self.bytes_fetched.add(metrics.metric_name('bytes-consumed-rate', self.group_name,

'The average number of bytes consumed per second'), Rate())

self.records_fetched = self.metrics.sensor('records-fetched')

self.records_fetched.add(metrics.metric_name('records-per-request-avg', self.group_name,

'The average number of records in each request'), Avg())

self.records_fetched.add(metrics.metric_name('records-consumed-rate', self.group_name,

'The average number of records consumed per second'), Rate())

self.fetch_latency = metrics.sensor('fetch-latency')

self.fetch_latency.add(metrics.metric_name('fetch-latency-avg', self.group_name,

'The average time taken for a fetch request.'), Avg())

self.fetch_latency.add(metrics.metric_name('fetch-latency-max', self.group_name,

'The max time taken for any fetch request.'), Max())

self.fetch_latency.add(metrics.metric_name('fetch-rate', self.group_name,

'The number of fetch requests per second.'), Rate(sampled_stat=Count()))

self.records_fetch_lag = metrics.sensor('records-lag')

self.records_fetch_lag.add(metrics.metric_name('records-lag-max', self.group_name,

'The maximum lag in terms of number of records for any partition in self window'), Max())

self.fetch_throttle_time_sensor = metrics.sensor('fetch-throttle-time')

self.fetch_throttle_time_sensor.add(metrics.metric_name('fetch-throttle-time-avg', self.group_name,

'The average throttle time in ms'), Avg())

self.fetch_throttle_time_sensor.add(metrics.metric_name('fetch-throttle-time-max', self.group_name,

'The maximum throttle time in ms'), Max())

def record_topic_fetch_metrics(self, topic, num_bytes, num_records):

# record bytes fetched

name = '.'.join(['topic', topic, 'bytes-fetched'])

bytes_fetched = self.metrics.get_sensor(name)

if not bytes_fetched:

metric_tags = {'topic': topic.replace('.', '_')}

bytes_fetched = self.metrics.sensor(name)

bytes_fetched.add(self.metrics.metric_name('fetch-size-avg',

self.group_name,

'The average number of bytes fetched per request for topic %s' % topic,

metric_tags), Avg())

bytes_fetched.add(self.metrics.metric_name('fetch-size-max',

self.group_name,

'The maximum number of bytes fetched per request for topic %s' % topic,

metric_tags), Max())

bytes_fetched.add(self.metrics.metric_name('bytes-consumed-rate',

self.group_name,

'The average number of bytes consumed per second for topic %s' % topic,

metric_tags), Rate())

bytes_fetched.record(num_bytes)

# record records fetched

name = '.'.join(['topic', topic, 'records-fetched'])

records_fetched = self.metrics.get_sensor(name)

if not records_fetched:

metric_tags = {'topic': topic.replace('.', '_')}

records_fetched = self.metrics.sensor(name)

records_fetched.add(self.metrics.metric_name('records-per-request-avg',

self.group_name,

'The average number of records in each request for topic %s' % topic,

metric_tags), Avg())

records_fetched.add(self.metrics.metric_name('records-consumed-rate',

self.group_name,

'The average number of records consumed per second for topic %s' % topic,

metric_tags), Rate())