def determine_offsets(self, topic, start=None):
"""
Sends OffsetRequests to the cluster for a given topic and start point.
The ``start`` parameter can be any of of ``datetime.datetime``,
``datetime.timedelta`` or one of `SingleConsumer.BEGINNING` or
`SingleConsumer.END`. The value is translated into epoch seconds
if need be and used for the "time" parameter for the offset requests.
An offset request is sent to each of the leader brokers for the given
topic.
"""
log.info("Getting offsets for topic %s with start %s", topic, start)
if start is None:
start = self.END
offset_time = start_to_timestamp(start)
def request_factory():
return offset.OffsetRequest(
replica_id=CONSUMER_REPLICA_ID,
topics=[offset.TopicRequest(name=topic, partitions=[])])
requests = collections.defaultdict(request_factory)
for partition_id in self.allocation[topic]:
leader = self.cluster.get_leader(topic, partition_id)
requests[leader].topics[0].partitions.append(
offset.PartitionRequest(partition_id=partition_id,
time=offset_time,
max_offsets=1))
log.debug("Sending offset request to %d leaders.",
len(requests.keys()))
yield self.send(requests)
raise gen.Return(True)
def test_default_consumes_from_end_offset(self):
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="offset",
responses=[
offset.OffsetResponse(
topics=[
offset.TopicResponse(
name="test.topic",
partitions=[
offset.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offsets=[99],
)
]
)
]
)
]
)
self.set_responses(
broker_id=1, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"cat": "meow"}',
)
),
(
1,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"dog": "bark"}',
)
),
]
)
),
]
),
]
),
]
)
c = single.SingleConsumer(["kafka01"])
yield c.connect()
msgs = yield c.consume("test.topic")
yield c.close()
self.assertEqual(msgs, [{"cat": "meow"}, {"dog": "bark"}])
self.assert_sent(
broker_id=1,
request=offset.OffsetRequest(
replica_id=-1,
topics=[
offset.TopicRequest(
name="test.topic",
partitions=[
offset.PartitionRequest(
partition_id=0,
time=-1, # alias for 'end of topic'
max_offsets=1,
)
]
)
]
)
)
self.assert_sent(
broker_id=1,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=0,
offset=99,
max_bytes=(1024 * 1024),
),
]
)
]
)
)
def test_timedelta_offset(self, mock_calendar):
# captured the epoch seconds when first writing this
written_epoch = 1455849320
mock_calendar.timegm.return_value = written_epoch
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="offset",
responses=[
offset.OffsetResponse(
topics=[
offset.TopicResponse(
name="test.topic",
partitions=[
offset.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offsets=[99],
)
]
)
]
)
]
)
self.set_responses(
broker_id=1, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet([]),
),
]
),
]
),
]
)
c = single.SingleConsumer(["kafka01"])
yield c.connect()
two_days = datetime.timedelta(days=2)
yield c.consume("test.topic", start=two_days)
self.assert_sent(
broker_id=1,
request=offset.OffsetRequest(
replica_id=-1,
topics=[
offset.TopicRequest(
name="test.topic",
partitions=[
offset.PartitionRequest(
partition_id=0,
# two days *in the past*
time=written_epoch - two_days.total_seconds(),
max_offsets=1,
)
]
)
]
)
)
def test_datetime_offset(self):
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="offset",
responses=[
offset.OffsetResponse(
topics=[
offset.TopicResponse(
name="test.topic",
partitions=[
offset.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offsets=[99],
)
]
)
]
)
]
)
self.set_responses(
broker_id=1, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet([]),
),
]
),
]
),
]
)
c = single.SingleConsumer(["kafka01"])
yield c.connect()
start = datetime.datetime(2016, 2, 18, 0, 0)
secs_since_epoch = (
start - datetime.datetime(1970, 1, 1, 0, 0)
).total_seconds()
yield c.consume("test.topic", start=start)
self.assert_sent(
broker_id=1,
request=offset.OffsetRequest(
replica_id=-1,
topics=[
offset.TopicRequest(
name="test.topic",
partitions=[
offset.PartitionRequest(
partition_id=0,
time=secs_since_epoch,
max_offsets=1,
)
]
)
]
)
)
def test_beginning_offset(self):
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="offset",
responses=[
offset.OffsetResponse(
topics=[
offset.TopicResponse(
name="test.topic",
partitions=[
offset.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offsets=[99],
)
]
)
]
)
]
)
self.set_responses(
broker_id=1, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet([]),
),
]
),
]
),
]
)
c = single.SingleConsumer(["kafka01"])
yield c.connect()
yield c.consume("test.topic", start=single.SingleConsumer.BEGINNING)
self.assert_sent(
broker_id=1,
request=offset.OffsetRequest(
replica_id=-1,
topics=[
offset.TopicRequest(
name="test.topic",
partitions=[
offset.PartitionRequest(
partition_id=0,
time=-2, # alias for 'beginning of topic'
max_offsets=1,
)
]
)
]
)
)