def test_offset_behavior__resuming_behavior(self):
msgs1 = self.send_messages(0, range(0, 100))
msgs2 = self.send_messages(1, range(100, 200))
# Start a consumer
consumer1 = self.consumer(
auto_commit_every_t = None,
auto_commit_every_n = 20,
)
# Grab the first 195 messages
output_msgs1 = [ consumer1.get_message().message.value for _ in compat.xrange(195) ]
self.assert_message_count(output_msgs1, 195)
# The total offset across both partitions should be at 180
consumer2 = self.consumer(
auto_commit_every_t = None,
auto_commit_every_n = 20,
)
# 181-200
self.assert_message_count([ message for message in consumer2 ], 20)
consumer1.stop()
consumer2.stop()
def test_offset_behavior__resuming_behavior(self):
msgs1 = self.send_messages(0, range(0, 100))
msgs2 = self.send_messages(1, range(100, 200))
# Start a consumer
consumer1 = self.consumer(
auto_commit_every_t=None,
auto_commit_every_n=20,
)
# Grab the first 195 messages
output_msgs1 = [
consumer1.get_message().message.value for _ in compat.xrange(195)
]
self.assert_message_count(output_msgs1, 195)
# The total offset across both partitions should be at 180
consumer2 = self.consumer(
auto_commit_every_t=None,
auto_commit_every_n=20,
)
# 181-200
self.assert_message_count([message for message in consumer2], 20)
consumer1.stop()
consumer2.stop()
def decode_offset_commit_response(cls, data):
"""
Decode bytes to an OffsetCommitResponse
Params
======
data: bytes to decode
"""
((correlation_id,), cur) = relative_unpack('>i', data, 0)
((num_topics,), cur) = relative_unpack('>i', data, cur)
for i in compat.xrange(num_topics):
(topic, cur) = read_short_string(data, cur)
((num_partitions,), cur) = relative_unpack('>i', data, cur)
for i in compat.xrange(num_partitions):
((partition, error), cur) = relative_unpack('>ih', data, cur)
yield OffsetCommitResponse(topic, partition, error)
def test_snappy(self):
for i in compat.xrange(1000):
s1 = compat.bytes(random_string(100))
s2 = snappy_decode(snappy_encode(s1))
self.assertEquals(s1, s2)
def test_gzip(self):
for i in compat.xrange(1000):
s1 = compat.bytes(random_string(100))
s2 = gzip_decode(gzip_encode(s1))
self.assertEquals(s1, s2)
def random_string(l):
s = "".join(random.choice(compat.letters) for i in compat.xrange(l))
return s
def _chunker():
for i in compat.xrange(0, len(payload), xerial_blocksize):
yield payload[i:i + xerial_blocksize]
def random_string(l):
s = "".join(random.choice(compat.letters) for i in compat.xrange(l))
return s
def test_snappy(self):
for i in compat.xrange(1000):
s1 = compat.bytes(random_string(100))
s2 = snappy_decode(snappy_encode(s1))
self.assertEquals(s1, s2)
def test_gzip(self):
for i in compat.xrange(1000):
s1 = compat.bytes(random_string(100))
s2 = gzip_decode(gzip_encode(s1))
self.assertEquals(s1, s2)
def _chunker():
for i in compat.xrange(0, len(payload), xerial_blocksize):
yield payload[i : i + xerial_blocksize]
