def test_produce_many_snappy(self):
start_offset = yield from self.current_offset(self.topic, 0)
msgs1 = create_snappy_message(
[b"Snappy 1" + bytes(i) for i in range(100)])
msgs2 = create_snappy_message(
[b"Snappy 2" + bytes(i) for i in range(100)])
yield from self.assert_produce_request([msgs1, msgs2], start_offset,
200)
def test_produce_many_snappy(self):
start_offset = yield from self.current_offset(self.topic, 0)
msgs1 = create_snappy_message(
[b"Snappy 1" + bytes(i) for i in range(100)])
msgs2 = create_snappy_message(
[b"Snappy 2" + bytes(i) for i in range(100)])
yield from self.assert_produce_request([msgs1, msgs2], start_offset,
200)
def test_produce_many_snappy(self):
self.skipTest("All snappy integration tests fail with nosnappyjava")
start_offset = self.current_offset(self.topic, 0)
self.assert_produce_request([
create_snappy_message([("Snappy 1 %d" % i, None) for i in range(100)]),
create_snappy_message([("Snappy 2 %d" % i, None) for i in range(100)]),
],
start_offset,
200,
)
def test_produce_many_snappy(self):
self.skipTest("All snappy integration tests fail with nosnappyjava")
start_offset = self.current_offset(self.topic, 0)
self.assert_produce_request([
create_snappy_message([("Snappy 1 %d" % i, None) for i in range(100)]),
create_snappy_message([("Snappy 2 %d" % i, None) for i in range(100)]),
],
start_offset,
200,
)
def test_produce_mixed(self):
start_offset = yield from self.current_offset(self.topic, 0)
msg_count = 1 + 100 + 100
messages = [
create_message(b"Just a plain message"),
create_gzip_message([
("Gzipped %d" % i).encode('utf-8') for i in range(100)]),
create_snappy_message([
b"Snappy " + bytes(i) for i in range(100)])]
yield from self.assert_produce_request(messages, start_offset,
msg_count)
def test_produce_mixed(self):
start_offset = yield from self.current_offset(self.topic, 0)
msg_count = 1 + 100 + 100
messages = [
create_message(b"Just a plain message"),
create_gzip_message([("Gzipped %d" % i).encode('utf-8')
for i in range(100)]),
create_snappy_message([b"Snappy " + bytes(i) for i in range(100)])
]
yield from self.assert_produce_request(messages, start_offset,
msg_count)
def test_produce_mixed(self):
start_offset = self.current_offset(self.topic, 0)
msg_count = 1+100
messages = [
create_message(b"Just a plain message"),
create_gzip_message([
(("Gzipped %d" % i).encode('utf-8'), None) for i in range(100)]),
]
# All snappy integration tests fail with nosnappyjava
if False and has_snappy():
msg_count += 100
messages.append(create_snappy_message([("Snappy %d" % i, None) for i in range(100)]))
self.assert_produce_request(messages, start_offset, msg_count)
def test_produce_mixed(self):
start_offset = self.current_offset(self.topic, 0)
msg_count = 1+100
messages = [
create_message(b"Just a plain message"),
create_gzip_message([
(("Gzipped %d" % i).encode('utf-8'), None) for i in range(100)]),
]
# All snappy integration tests fail with nosnappyjava
if False and has_snappy():
msg_count += 100
messages.append(create_snappy_message([("Snappy %d" % i, None) for i in range(100)]))
self.assert_produce_request(messages, start_offset, msg_count)
from collections import namedtuple
import functools
from kafka.protocol import KafkaProtocol
from kafka import (
SimpleProducer, KeyedProducer,
create_message, create_gzip_message, create_snappy_message,
RoundRobinPartitioner, HashedPartitioner
)
ProduceRequestPayload = namedtuple("ProduceRequestPayload",
["topic", "partition", "messages"])
messages = [
create_snappy_message([("---start---", None), ("xxxxxxxxxxxxxxxxx", None)]),
create_snappy_message([("ccccc", None), ("---end---", None)]),
]
produce = ProduceRequestPayload("topic1", 1, messages=messages)
correlationID = 3
req = KafkaProtocol.encode_produce_request("clientID", correlationID, payloads=[produce], acks=1, timeout=1000)
print "[]byte{" +''.join( [ "0x%02x, " % ord( x ) for x in req ] ).strip() + "}"
# python test.py | hexdump -C
import subprocess
task = subprocess.Popen("hexdump -C", shell=True, stdin=subprocess.PIPE)
task.stdin.write(req)
task.stdin.flush()
