def test_produce_many_gzip(self):
start_offset = yield from self.current_offset(self.topic, 0)
message1 = create_gzip_message([
("Gzipped 1 %d" % i).encode('utf-8') for i in range(100)])
message2 = create_gzip_message([
("Gzipped 2 %d" % i).encode('utf-8') for i in range(100)])
yield from self.assert_produce_request([message1, message2],
start_offset, 200)
def test_produce_many_gzip(self):
start_offset = yield from self.current_offset(self.topic, 0)
message1 = create_gzip_message([("Gzipped 1 %d" % i).encode('utf-8')
for i in range(100)])
message2 = create_gzip_message([("Gzipped 2 %d" % i).encode('utf-8')
for i in range(100)])
yield from self.assert_produce_request([message1, message2],
start_offset, 200)
def test_produce_100k_gzipped(self):
start_offset = yield from self.current_offset(self.topic, 0)
msgs1 = [("Gzipped batch 1, message %d" % i).encode('utf-8')
for i in range(50000)]
yield from self.assert_produce_request([create_gzip_message(msgs1)],
start_offset, 50000, )
msgs2 = [("Gzipped batch 1, message %d" % i).encode('utf-8')
for i in range(50000)]
yield from self.assert_produce_request([create_gzip_message(msgs2)],
start_offset + 50000, 50000, )
def send_gzip_message(self, partition, messages):
message = create_gzip_message([(self.msg(str(msg)), None)
for msg in messages])
produce = ProduceRequestPayload(self.topic,
partition,
messages=[message])
resp, = self.client.send_produce_request([produce])
self.assertEqual(resp.error, 0)
def test_produce_100k_gzipped(self):
start_offset = yield from self.current_offset(self.topic, 0)
msgs1 = [("Gzipped batch 1, message %d" % i).encode('utf-8')
for i in range(50000)]
yield from self.assert_produce_request(
[create_gzip_message(msgs1)],
start_offset,
50000,
)
msgs2 = [("Gzipped batch 1, message %d" % i).encode('utf-8')
for i in range(50000)]
yield from self.assert_produce_request(
[create_gzip_message(msgs2)],
start_offset + 50000,
50000,
)
def test_produce_100k_gzipped(self):
start_offset = self.current_offset(self.topic, 0)
self.assert_produce_request([
create_gzip_message([
(("Gzipped batch 1, message %d" % i).encode('utf-8'), None)
for i in range(50000)])
],
start_offset,
50000,
)
self.assert_produce_request([
create_gzip_message([
(("Gzipped batch 1, message %d" % i).encode('utf-8'), None)
for i in range(50000)])
],
start_offset+50000,
50000,
)
def test_produce_100k_gzipped(self):
start_offset = self.current_offset(self.topic, 0)
self.assert_produce_request([
create_gzip_message([
(("Gzipped batch 1, message %d" % i).encode('utf-8'), None)
for i in range(50000)])
],
start_offset,
50000,
)
self.assert_produce_request([
create_gzip_message([
(("Gzipped batch 1, message %d" % i).encode('utf-8'), None)
for i in range(50000)])
],
start_offset+50000,
50000,
)
def test_produce_mixed(self):
start_offset = yield from self.current_offset(self.topic, 0)
msg_count = 1 + 100 + 100
messages = [
create_message(b"Just a plain message"),
create_gzip_message([
("Gzipped %d" % i).encode('utf-8') for i in range(100)]),
create_snappy_message([
b"Snappy " + bytes(i) for i in range(100)])]
yield from self.assert_produce_request(messages, start_offset,
msg_count)
def test_produce_mixed(self):
start_offset = yield from self.current_offset(self.topic, 0)
msg_count = 1 + 100 + 100
messages = [
create_message(b"Just a plain message"),
create_gzip_message([("Gzipped %d" % i).encode('utf-8')
for i in range(100)]),
create_snappy_message([b"Snappy " + bytes(i) for i in range(100)])
]
yield from self.assert_produce_request(messages, start_offset,
msg_count)
def test_produce_mixed(self):
start_offset = self.current_offset(self.topic, 0)
msg_count = 1+100
messages = [
create_message(b"Just a plain message"),
create_gzip_message([
(("Gzipped %d" % i).encode('utf-8'), None) for i in range(100)]),
]
# All snappy integration tests fail with nosnappyjava
if False and has_snappy():
msg_count += 100
messages.append(create_snappy_message([("Snappy %d" % i, None) for i in range(100)]))
self.assert_produce_request(messages, start_offset, msg_count)
def test_produce_mixed(self):
start_offset = self.current_offset(self.topic, 0)
msg_count = 1+100
messages = [
create_message(b"Just a plain message"),
create_gzip_message([
(("Gzipped %d" % i).encode('utf-8'), None) for i in range(100)]),
]
# All snappy integration tests fail with nosnappyjava
if False and has_snappy():
msg_count += 100
messages.append(create_snappy_message([("Snappy %d" % i, None) for i in range(100)]))
self.assert_produce_request(messages, start_offset, msg_count)
def send_gzip_message(self, partition, messages):
message = create_gzip_message([(self.msg(str(msg)), None) for msg in messages])
produce = ProduceRequestPayload(self.topic, partition, messages = [message])
resp, = self.client.send_produce_request([produce])
self.assertEqual(resp.error, 0)