def test_produce_mixed(self):
start_offset = yield self.current_offset(self.topic, 0)
msg_count = 1+100
messages = [
create_message("Just a plain message"),
create_message_set(
make_send_requests(
["Gzipped %d" % i for i in range(100)]), CODEC_GZIP)[0]
]
# Can't produce snappy messages without snappy...
if has_snappy():
msg_count += 100
messages.append(
create_message_set(
make_send_requests(["Snappy %d" % i for i in range(100)]),
CODEC_SNAPPY)[0])
yield self.assert_produce_request(messages, start_offset, msg_count)
ptMsgs = ['Just a plain message']
ptMsgs.extend(["Gzipped %d" % i for i in range(100)])
if has_snappy():
ptMsgs.extend(["Snappy %d" % i for i in range(100)])
yield self.assert_fetch_offset(0, start_offset, ptMsgs,
fetch_size=10240)
def test_produce_mixed(self):
start_offset = yield self.current_offset(self.topic, 0)
msg_count = 1 + 100
messages = [
create_message("Just a plain message"),
create_message_set(
make_send_requests(["Gzipped %d" % i for i in range(100)]),
CODEC_GZIP)[0]
]
# Can't produce snappy messages without snappy...
if has_snappy():
msg_count += 100
messages.append(
create_message_set(
make_send_requests(["Snappy %d" % i for i in range(100)]),
CODEC_SNAPPY)[0])
yield self.assert_produce_request(messages, start_offset, msg_count)
ptMsgs = ['Just a plain message']
ptMsgs.extend(["Gzipped %d" % i for i in range(100)])
if has_snappy():
ptMsgs.extend(["Snappy %d" % i for i in range(100)])
yield self.assert_fetch_offset(0,
start_offset,
ptMsgs,
fetch_size=10240)
def test_produce_10k_gzipped(self):
start_offset = yield self.current_offset(self.topic, 0)
msgs = create_message_set(
make_send_requests(
["Gzipped batch 1, message %d" % i for i in range(5000)]),
CODEC_GZIP)
yield self.assert_produce_request(msgs, start_offset, 5000)
msgs = create_message_set(
make_send_requests(
["Gzipped batch 2, message %d" % i for i in range(5000)]),
CODEC_GZIP)
yield self.assert_produce_request(msgs, start_offset + 5000, 5000)
def test_produce_10k_gzipped(self):
start_offset = yield self.current_offset(self.topic, 0)
msgs = create_message_set(
make_send_requests(
[b"Gzipped batch 1, message %d" % i for i in range(5000)]),
CODEC_GZIP)
yield self.assert_produce_request(msgs, start_offset, 5000)
msgs = create_message_set(
make_send_requests(
[b"Gzipped batch 2, message %d" % i for i in range(5000)]),
CODEC_GZIP)
yield self.assert_produce_request(msgs, start_offset+5000, 5000)
def test_produce_many_snappy(self):
start_offset = yield self.current_offset(self.topic, 0)
message1 = create_message_set(
make_send_requests(["Snappy 1 %d" % i for i in range(100)]),
CODEC_SNAPPY)[0]
message2 = create_message_set(
make_send_requests(["Snappy 2 %d" % i for i in range(100)]),
CODEC_SNAPPY)[0]
yield self.assert_produce_request(
[message1, message2],
start_offset,
200,
)
def test_produce_many_snappy(self):
start_offset = yield self.current_offset(self.topic, 0)
message1 = create_message_set(
make_send_requests(
[b"Snappy 1 %d" % i for i in range(100)]), CODEC_SNAPPY)[0]
message2 = create_message_set(
make_send_requests(
[b"Snappy 2 %d" % i for i in range(100)]), CODEC_SNAPPY)[0]
yield self.assert_produce_request(
[message1, message2],
start_offset,
200,
)
def test_produce_many_gzip(self):
start_offset = yield self.current_offset(self.topic, 0)
message1 = create_message_set(
make_send_requests(["Gzipped 1 %d" % i for i in range(100)]),
CODEC_GZIP)[0]
message2 = create_message_set(
make_send_requests(["Gzipped 2 %d" % i for i in range(100)]),
CODEC_GZIP)[0]
yield self.assert_produce_request(
[message1, message2],
start_offset,
200,
)
def test_produce_many_gzip(self):
start_offset = yield self.current_offset(self.topic, 0)
message1 = create_message_set(
make_send_requests(
[b"Gzipped 1 %d" % i for i in range(100)]), CODEC_GZIP)[0]
message2 = create_message_set(
make_send_requests(
[b"Gzipped 2 %d" % i for i in range(100)]), CODEC_GZIP)[0]
yield self.assert_produce_request(
[message1, message2],
start_offset,
200,
)
def test_produce_100_keyed_gzipped(self):
"""test_produce_100_keyed_gzipped
Test that gzipping the batch doesn't affect the partition to which
the messages are assigned, or the order of the messages in the topics
"""
start_offset = yield self.current_offset(self.topic, 0)
msg_set = create_message_set([
SendRequest(self.topic, "Key:%d" % i, ["Test msg %d" % i], None)
for i in range(100)
], CODEC_GZIP)
yield self.assert_produce_request(
msg_set,
start_offset,
100,
)
msgs = ["Test msg %d" % i for i in range(100)]
keys = ["Key:%d" % i for i in range(100)]
yield self.assert_fetch_offset(0,
start_offset,
msgs,
expected_keys=keys,
fetch_size=10240)
def test_produce_100_keyed_gzipped(self):
"""test_produce_100_keyed_gzipped
Test that gzipping the batch doesn't affect the partition to which
the messages are assigned, or the order of the messages in the topics
"""
start_offset = yield self.current_offset(self.topic, 0)
msg_set = create_message_set(
[SendRequest(self.topic, b"Key:%d" % i,
[b"Test msg %d" % i], None)
for i in range(100)], CODEC_GZIP)
yield self.assert_produce_request(
msg_set,
start_offset,
100,
)
msgs = [b"Test msg %d" % i for i in range(100)]
keys = [b"Key:%d" % i for i in range(100)]
yield self.assert_fetch_offset(0, start_offset, msgs,
expected_keys=keys,
fetch_size=10240)
