def test_check_offset(self):
# Produce/consume a message, check that the next offset looks correct
message1 = KafkaClient.create_message("testing 1")
req = ProduceRequest("test-check-offset", 0, [message1])
self.kafka.send_message_set(req)
self.assertTrue(
self.server.wait_for("Created log for 'test-check-offset'-0"))
self.assertTrue(
self.server.wait_for("Flushing log 'test-check-offset-0'"))
req = FetchRequest("test-check-offset", 0, 0, 1024)
(messages, nextReq) = self.kafka.get_message_set(req)
self.assertEquals(len(messages), 1)
self.assertEquals(messages[0], message1)
self.assertEquals(nextReq.offset,
len(KafkaClient.encode_message(message1)))
# Produce another message, consume with the last offset
message2 = KafkaClient.create_message("test 2")
req = ProduceRequest("test-check-offset", 0, [message2])
self.kafka.send_message_set(req)
self.assertTrue(
self.server.wait_for("Flushing log 'test-check-offset-0'"))
# Verify
(messages, nextReq) = self.kafka.get_message_set(nextReq)
self.assertEquals(len(messages), 1)
self.assertEquals(messages[0], message2)
self.assertEquals(
nextReq.offset,
len(KafkaClient.encode_message(message1)) +
len(KafkaClient.encode_message(message2)))
def test_check_offset(self):
# Produce/consume a message, check that the next offset looks correct
message1 = KafkaClient.create_message("testing 1")
req = ProduceRequest("test-check-offset", 0, [message1])
self.kafka.send_message_set(req)
self.assertTrue(self.server.wait_for("Created log for 'test-check-offset'-0"))
self.assertTrue(self.server.wait_for("Flushing log 'test-check-offset-0'"))
req = FetchRequest("test-check-offset", 0, 0, 1024)
(messages, nextReq) = self.kafka.get_message_set(req)
self.assertEquals(len(messages), 1)
self.assertEquals(messages[0], message1)
self.assertEquals(nextReq.offset, len(KafkaClient.encode_message(message1)))
# Produce another message, consume with the last offset
message2 = KafkaClient.create_message("test 2")
req = ProduceRequest("test-check-offset", 0, [message2])
self.kafka.send_message_set(req)
self.assertTrue(self.server.wait_for("Flushing log 'test-check-offset-0'"))
# Verify
(messages, nextReq) = self.kafka.get_message_set(nextReq)
self.assertEquals(len(messages), 1)
self.assertEquals(messages[0], message2)
self.assertEquals(
nextReq.offset, len(KafkaClient.encode_message(message1)) + len(KafkaClient.encode_message(message2))
)
def test_message_simple(self):
msg = KafkaClient.create_message("testing")
enc = KafkaClient.encode_message(msg)
expect = "\x00\x00\x00\r\x01\x00\xe8\xf3Z\x06testing"
self.assertEquals(enc, expect)
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), 1)
self.assertEquals(messages[0], msg)
def test_message_simple(self):
msg = KafkaClient.create_message("testing")
enc = KafkaClient.encode_message(msg)
expect = "\x00\x00\x00\r\x01\x00\xe8\xf3Z\x06testing"
self.assertEquals(enc, expect)
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), 1)
self.assertEquals(messages[0], msg)
def test_message_snappy(self):
msg = KafkaClient.create_snappy_message("one", "two", "three")
enc = KafkaClient.encode_message(msg)
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), 3)
self.assertEquals(messages[0].payload, "one")
self.assertEquals(messages[1].payload, "two")
self.assertEquals(messages[2].payload, "three")
def test_message_snappy(self):
msg = KafkaClient.create_snappy_message("one", "two", "three")
enc = KafkaClient.encode_message(msg)
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), 3)
self.assertEquals(messages[0].payload, "one")
self.assertEquals(messages[1].payload, "two")
self.assertEquals(messages[2].payload, "three")
def test_message_gzip(self):
msg = KafkaClient.create_gzip_message("one", "two", "three")
enc = KafkaClient.encode_message(msg)
# Can't check the bytes directly since Gzip is non-deterministic
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), 3)
self.assertEquals(messages[0].payload, "one")
self.assertEquals(messages[1].payload, "two")
self.assertEquals(messages[2].payload, "three")
def test_message_gzip(self):
msg = KafkaClient.create_gzip_message("one", "two", "three")
enc = KafkaClient.encode_message(msg)
# Can't check the bytes directly since Gzip is non-deterministic
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), 3)
self.assertEquals(messages[0].payload, "one")
self.assertEquals(messages[1].payload, "two")
self.assertEquals(messages[2].payload, "three")
def test_message_gzip_random(self):
for i in xrange(ITERATIONS):
n = random.randint(0, 10)
strings = [random_string() for j in range(n)]
msg = KafkaClient.create_gzip_message(*strings)
enc = KafkaClient.encode_message(msg)
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), n)
for j in range(n):
self.assertEquals(messages[j].payload, strings[j])
def test_message_gzip_random(self):
for i in xrange(ITERATIONS):
n = random.randint(0, 10)
strings = [random_string() for j in range(n)]
msg = KafkaClient.create_gzip_message(*strings)
enc = KafkaClient.encode_message(msg)
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), n)
for j in range(n):
self.assertEquals(messages[j].payload, strings[j])
