def test_10k_messages(self):
msg_tmpl = "this is a test message with a few bytes in it. this is message number %d"
# TODO 10k actually fails, why?
msg = KafkaClient.create_gzip_message(*[msg_tmpl % i for i in range(1000)])
req = ProduceRequest("test-10k", 0, [msg])
self.kafka.send_message_set(req)
self.assertTrue(self.server.wait_for("Created log for 'test-10k'-0"))
self.assertTrue(self.server.wait_for("Flushing log 'test-10k-0'"))
def test_message_gzip(self):
msg = KafkaClient.create_gzip_message("one", "two", "three")
enc = KafkaClient.encode_message(msg)
# Can't check the bytes directly since Gzip is non-deterministic
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), 3)
self.assertEquals(messages[0].payload, "one")
self.assertEquals(messages[1].payload, "two")
self.assertEquals(messages[2].payload, "three")
def test_message_gzip(self):
msg = KafkaClient.create_gzip_message("one", "two", "three")
enc = KafkaClient.encode_message(msg)
# Can't check the bytes directly since Gzip is non-deterministic
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), 3)
self.assertEquals(messages[0].payload, "one")
self.assertEquals(messages[1].payload, "two")
self.assertEquals(messages[2].payload, "three")
def test_10k_messages(self):
msg_tmpl = "this is a test message with a few bytes in it. this is message number %d"
# TODO 10k actually fails, why?
msg = KafkaClient.create_gzip_message(
*[msg_tmpl % i for i in range(1000)])
req = ProduceRequest("test-10k", 0, [msg])
self.kafka.send_message_set(req)
self.assertTrue(self.server.wait_for("Created log for 'test-10k'-0"))
self.assertTrue(self.server.wait_for("Flushing log 'test-10k-0'"))
def test_message_gzip_random(self):
for i in xrange(ITERATIONS):
n = random.randint(0, 10)
strings = [random_string() for j in range(n)]
msg = KafkaClient.create_gzip_message(*strings)
enc = KafkaClient.encode_message(msg)
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), n)
for j in range(n):
self.assertEquals(messages[j].payload, strings[j])
def test_message_gzip_random(self):
for i in xrange(ITERATIONS):
n = random.randint(0, 10)
strings = [random_string() for j in range(n)]
msg = KafkaClient.create_gzip_message(*strings)
enc = KafkaClient.encode_message(msg)
(messages, read) = KafkaClient.read_message_set(enc)
self.assertEquals(len(messages), n)
for j in range(n):
self.assertEquals(messages[j].payload, strings[j])
def test_create_gzip(self):
msg = KafkaClient.create_gzip_message("testing")
self.assertEquals(msg.magic, 1)
self.assertEquals(msg.attributes, 1)
# Can't check the crc or payload for gzip since it's non-deterministic
(messages, _) = KafkaClient.read_message_set(gzip_decode(msg.payload))
inner = messages[0]
self.assertEquals(inner.magic, 1)
self.assertEquals(inner.attributes, 0)
self.assertEquals(inner.payload, "testing")
self.assertEquals(inner.crc, -386704890)
def test_create_gzip(self):
msg = KafkaClient.create_gzip_message("testing")
self.assertEquals(msg.magic, 1)
self.assertEquals(msg.attributes, 1)
# Can't check the crc or payload for gzip since it's non-deterministic
(messages, _) = KafkaClient.read_message_set(gzip_decode(msg.payload))
inner = messages[0]
self.assertEquals(inner.magic, 1)
self.assertEquals(inner.attributes, 0)
self.assertEquals(inner.payload, "testing")
self.assertEquals(inner.crc, -386704890)
