def test_produce_mixed(self): start_offset = yield self.current_offset(self.topic, 0) msg_count = 1+100 messages = [ create_message("Just a plain message"), create_message_set( make_send_requests( ["Gzipped %d" % i for i in range(100)]), CODEC_GZIP)[0] ] # Can't produce snappy messages without snappy... if has_snappy(): msg_count += 100 messages.append( create_message_set( make_send_requests(["Snappy %d" % i for i in range(100)]), CODEC_SNAPPY)[0]) yield self.assert_produce_request(messages, start_offset, msg_count) ptMsgs = ['Just a plain message'] ptMsgs.extend(["Gzipped %d" % i for i in range(100)]) if has_snappy(): ptMsgs.extend(["Snappy %d" % i for i in range(100)]) yield self.assert_fetch_offset(0, start_offset, ptMsgs, fetch_size=10240)
def test_produce_mixed(self): start_offset = yield self.current_offset(self.topic, 0) msg_count = 1 + 100 messages = [ create_message("Just a plain message"), create_message_set( make_send_requests(["Gzipped %d" % i for i in range(100)]), CODEC_GZIP)[0] ] # Can't produce snappy messages without snappy... if has_snappy(): msg_count += 100 messages.append( create_message_set( make_send_requests(["Snappy %d" % i for i in range(100)]), CODEC_SNAPPY)[0]) yield self.assert_produce_request(messages, start_offset, msg_count) ptMsgs = ['Just a plain message'] ptMsgs.extend(["Gzipped %d" % i for i in range(100)]) if has_snappy(): ptMsgs.extend(["Snappy %d" % i for i in range(100)]) yield self.assert_fetch_offset(0, start_offset, ptMsgs, fetch_size=10240)
def test_produce_10k_gzipped(self): start_offset = yield self.current_offset(self.topic, 0) msgs = create_message_set( make_send_requests( ["Gzipped batch 1, message %d" % i for i in range(5000)]), CODEC_GZIP) yield self.assert_produce_request(msgs, start_offset, 5000) msgs = create_message_set( make_send_requests( ["Gzipped batch 2, message %d" % i for i in range(5000)]), CODEC_GZIP) yield self.assert_produce_request(msgs, start_offset + 5000, 5000)
def test_produce_10k_gzipped(self): start_offset = yield self.current_offset(self.topic, 0) msgs = create_message_set( make_send_requests( [b"Gzipped batch 1, message %d" % i for i in range(5000)]), CODEC_GZIP) yield self.assert_produce_request(msgs, start_offset, 5000) msgs = create_message_set( make_send_requests( [b"Gzipped batch 2, message %d" % i for i in range(5000)]), CODEC_GZIP) yield self.assert_produce_request(msgs, start_offset+5000, 5000)
def test_produce_many_snappy(self): start_offset = yield self.current_offset(self.topic, 0) message1 = create_message_set( make_send_requests(["Snappy 1 %d" % i for i in range(100)]), CODEC_SNAPPY)[0] message2 = create_message_set( make_send_requests(["Snappy 2 %d" % i for i in range(100)]), CODEC_SNAPPY)[0] yield self.assert_produce_request( [message1, message2], start_offset, 200, )
def test_produce_many_snappy(self): start_offset = yield self.current_offset(self.topic, 0) message1 = create_message_set( make_send_requests( [b"Snappy 1 %d" % i for i in range(100)]), CODEC_SNAPPY)[0] message2 = create_message_set( make_send_requests( [b"Snappy 2 %d" % i for i in range(100)]), CODEC_SNAPPY)[0] yield self.assert_produce_request( [message1, message2], start_offset, 200, )
def test_produce_many_gzip(self): start_offset = yield self.current_offset(self.topic, 0) message1 = create_message_set( make_send_requests(["Gzipped 1 %d" % i for i in range(100)]), CODEC_GZIP)[0] message2 = create_message_set( make_send_requests(["Gzipped 2 %d" % i for i in range(100)]), CODEC_GZIP)[0] yield self.assert_produce_request( [message1, message2], start_offset, 200, )
def test_produce_many_gzip(self): start_offset = yield self.current_offset(self.topic, 0) message1 = create_message_set( make_send_requests( [b"Gzipped 1 %d" % i for i in range(100)]), CODEC_GZIP)[0] message2 = create_message_set( make_send_requests( [b"Gzipped 2 %d" % i for i in range(100)]), CODEC_GZIP)[0] yield self.assert_produce_request( [message1, message2], start_offset, 200, )
def test_produce_100_keyed_gzipped(self): """test_produce_100_keyed_gzipped Test that gzipping the batch doesn't affect the partition to which the messages are assigned, or the order of the messages in the topics """ start_offset = yield self.current_offset(self.topic, 0) msg_set = create_message_set([ SendRequest(self.topic, "Key:%d" % i, ["Test msg %d" % i], None) for i in range(100) ], CODEC_GZIP) yield self.assert_produce_request( msg_set, start_offset, 100, ) msgs = ["Test msg %d" % i for i in range(100)] keys = ["Key:%d" % i for i in range(100)] yield self.assert_fetch_offset(0, start_offset, msgs, expected_keys=keys, fetch_size=10240)
def test_produce_100_keyed_gzipped(self): """test_produce_100_keyed_gzipped Test that gzipping the batch doesn't affect the partition to which the messages are assigned, or the order of the messages in the topics """ start_offset = yield self.current_offset(self.topic, 0) msg_set = create_message_set( [SendRequest(self.topic, b"Key:%d" % i, [b"Test msg %d" % i], None) for i in range(100)], CODEC_GZIP) yield self.assert_produce_request( msg_set, start_offset, 100, ) msgs = [b"Test msg %d" % i for i in range(100)] keys = [b"Key:%d" % i for i in range(100)] yield self.assert_fetch_offset(0, start_offset, msgs, expected_keys=keys, fetch_size=10240)