def test_produce_mixed(self):
start_offset = yield self.current_offset(self.topic, 0)
msg_count = 1+100
messages = [
create_message("Just a plain message"),
create_message_set(
make_send_requests(
["Gzipped %d" % i for i in range(100)]), CODEC_GZIP)[0]
]
# Can't produce snappy messages without snappy...
if has_snappy():
msg_count += 100
messages.append(
create_message_set(
make_send_requests(["Snappy %d" % i for i in range(100)]),
CODEC_SNAPPY)[0])
yield self.assert_produce_request(messages, start_offset, msg_count)
ptMsgs = ['Just a plain message']
ptMsgs.extend(["Gzipped %d" % i for i in range(100)])
if has_snappy():
ptMsgs.extend(["Snappy %d" % i for i in range(100)])
yield self.assert_fetch_offset(0, start_offset, ptMsgs,
fetch_size=10240)
def test_producer_send_messages_keyed_same_partition(self):
"""test_producer_send_messages_keyed_same_partition
Test that messages sent with a key are actually sent with that key,
even if they go to the same topic/partition (batching preserves keys)
"""
first_part = 43
second_part = 55
client = Mock()
ret1 = Deferred()
client.send_produce_request.side_effect = [ret1]
client.topic_partitions = {self.topic: [first_part, second_part]}
client.metadata_error_for_topic.return_value = False
msgs1 = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("odd_man_out")]
msgs3 = [self.msg("three"), self.msg("four")]
key1 = '99'
key3 = 'foo'
ack_timeout = 5
# Even though we're sending keyed messages, we use the default
# round-robin partitioner, since the requests are easier to predict
producer = Producer(client,
ack_timeout=ack_timeout,
batch_send=True,
batch_every_n=4)
d1 = producer.send_messages(self.topic, key=key1, msgs=msgs1)
d2 = producer.send_messages(self.topic, msgs=msgs2)
d3 = producer.send_messages(self.topic, key=key3, msgs=msgs3)
# Check the expected request was sent
msgSet1 = create_message_set([
make_send_requests(msgs1, key=key1)[0],
make_send_requests(msgs3, key=key3)[0]
], producer.codec)
msgSet2 = create_message_set(make_send_requests(msgs2), producer.codec)
req1 = ProduceRequest(self.topic, first_part, msgSet1)
req2 = ProduceRequest(self.topic, second_part, msgSet2)
# Annoying, but order of requests is indeterminate...
client.send_produce_request.assert_called_once_with(
ANY,
acks=producer.req_acks,
timeout=ack_timeout,
fail_on_error=False)
self.assertEqual(sorted([req1, req2]),
sorted(client.send_produce_request.call_args[0][0]))
# Check results when "response" fires
self.assertNoResult(d1)
self.assertNoResult(d2)
self.assertNoResult(d3)
resp = [
ProduceResponse(self.topic, first_part, 0, 10L),
ProduceResponse(self.topic, second_part, 0, 23L)
]
ret1.callback(resp)
result = self.successResultOf(d1)
self.assertEqual(result, resp[0])
result = self.successResultOf(d2)
self.assertEqual(result, resp[1])
result = self.successResultOf(d3)
self.assertEqual(result, resp[0])
producer.stop()
def test_produce_mixed(self):
start_offset = yield self.current_offset(self.topic, 0)
msg_count = 1 + 100
messages = [
create_message("Just a plain message"),
create_message_set(
make_send_requests(["Gzipped %d" % i for i in range(100)]),
CODEC_GZIP)[0]
]
# Can't produce snappy messages without snappy...
if has_snappy():
msg_count += 100
messages.append(
create_message_set(
make_send_requests(["Snappy %d" % i for i in range(100)]),
CODEC_SNAPPY)[0])
yield self.assert_produce_request(messages, start_offset, msg_count)
ptMsgs = ['Just a plain message']
ptMsgs.extend(["Gzipped %d" % i for i in range(100)])
if has_snappy():
ptMsgs.extend(["Snappy %d" % i for i in range(100)])
yield self.assert_fetch_offset(0,
start_offset,
ptMsgs,
fetch_size=10240)
def test_producer_send_messages_keyed_same_partition(self):
"""test_producer_send_messages_keyed_same_partition
Test that messages sent with a key are actually sent with that key,
even if they go to the same topic/partition (batching preserves keys)
"""
first_part = 43
second_part = 55
client = Mock()
ret1 = Deferred()
client.send_produce_request.side_effect = [ret1]
client.topic_partitions = {self.topic: [first_part, second_part]}
client.metadata_error_for_topic.return_value = False
msgs1 = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("odd_man_out")]
msgs3 = [self.msg("three"), self.msg("four")]
key1 = '99'
key3 = 'foo'
ack_timeout = 5
# Even though we're sending keyed messages, we use the default
# round-robin partitioner, since the requests are easier to predict
producer = Producer(client, ack_timeout=ack_timeout, batch_send=True,
batch_every_n=4)
d1 = producer.send_messages(self.topic, key=key1, msgs=msgs1)
d2 = producer.send_messages(self.topic, msgs=msgs2)
d3 = producer.send_messages(self.topic, key=key3, msgs=msgs3)
# Check the expected request was sent
msgSet1 = create_message_set(
[make_send_requests(msgs1, key=key1)[0],
make_send_requests(msgs3, key=key3)[0]], producer.codec)
msgSet2 = create_message_set(make_send_requests(
msgs2), producer.codec)
req1 = ProduceRequest(self.topic, first_part, msgSet1)
req2 = ProduceRequest(self.topic, second_part, msgSet2)
# Annoying, but order of requests is indeterminate...
client.send_produce_request.assert_called_once_with(
ANY, acks=producer.req_acks, timeout=ack_timeout,
fail_on_error=False)
self.assertEqual(sorted([req1, req2]),
sorted(client.send_produce_request.call_args[0][0]))
# Check results when "response" fires
self.assertNoResult(d1)
self.assertNoResult(d2)
self.assertNoResult(d3)
resp = [ProduceResponse(self.topic, first_part, 0, 10L),
ProduceResponse(self.topic, second_part, 0, 23L)]
ret1.callback(resp)
result = self.successResultOf(d1)
self.assertEqual(result, resp[0])
result = self.successResultOf(d2)
self.assertEqual(result, resp[1])
result = self.successResultOf(d3)
self.assertEqual(result, resp[0])
producer.stop()
def test_produce_10k_gzipped(self):
start_offset = yield self.current_offset(self.topic, 0)
msgs = create_message_set(
make_send_requests(
["Gzipped batch 1, message %d" % i for i in range(5000)]),
CODEC_GZIP)
yield self.assert_produce_request(msgs, start_offset, 5000)
msgs = create_message_set(
make_send_requests(
["Gzipped batch 2, message %d" % i for i in range(5000)]),
CODEC_GZIP)
yield self.assert_produce_request(msgs, start_offset + 5000, 5000)
def test_produce_10k_gzipped(self):
start_offset = yield self.current_offset(self.topic, 0)
msgs = create_message_set(
make_send_requests(
["Gzipped batch 1, message %d" % i for i in range(5000)]),
CODEC_GZIP)
yield self.assert_produce_request(msgs, start_offset, 5000)
msgs = create_message_set(
make_send_requests(
["Gzipped batch 2, message %d" % i for i in range(5000)]),
CODEC_GZIP)
yield self.assert_produce_request(msgs, start_offset+5000, 5000)
def test_produce_many_snappy(self):
start_offset = yield self.current_offset(self.topic, 0)
message1 = create_message_set(
make_send_requests(["Snappy 1 %d" % i for i in range(100)]),
CODEC_SNAPPY)[0]
message2 = create_message_set(
make_send_requests(["Snappy 2 %d" % i for i in range(100)]),
CODEC_SNAPPY)[0]
yield self.assert_produce_request(
[message1, message2],
start_offset,
200,
)
def test_produce_many_snappy(self):
start_offset = yield self.current_offset(self.topic, 0)
message1 = create_message_set(
make_send_requests(
["Snappy 1 %d" % i for i in range(100)]), CODEC_SNAPPY)[0]
message2 = create_message_set(
make_send_requests(
["Snappy 2 %d" % i for i in range(100)]), CODEC_SNAPPY)[0]
yield self.assert_produce_request(
[message1, message2],
start_offset,
200,
)
def test_produce_many_gzip(self):
start_offset = yield self.current_offset(self.topic, 0)
message1 = create_message_set(
make_send_requests(["Gzipped 1 %d" % i for i in range(100)]),
CODEC_GZIP)[0]
message2 = create_message_set(
make_send_requests(["Gzipped 2 %d" % i for i in range(100)]),
CODEC_GZIP)[0]
yield self.assert_produce_request(
[message1, message2],
start_offset,
200,
)
def test_produce_many_gzip(self):
start_offset = yield self.current_offset(self.topic, 0)
message1 = create_message_set(
make_send_requests(
["Gzipped 1 %d" % i for i in range(100)]), CODEC_GZIP)[0]
message2 = create_message_set(
make_send_requests(
["Gzipped 2 %d" % i for i in range(100)]), CODEC_GZIP)[0]
yield self.assert_produce_request(
[message1, message2],
start_offset,
200,
)
def test_producer_cancel_one_request_getting_topic(self):
# Test cancelling a request after it's begun to be processed
client = Mock()
client.topic_partitions = {}
ret = Deferred()
client.load_metadata_for_topics.return_value = ret
msgs = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
batch_n = 4
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
# This will trigger the metadata lookup
d2 = producer.send_messages(self.topic, msgs=msgs2)
d1.cancel()
self.failureResultOf(d1, CancelledError)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
self.assertNoResult(d2)
# Setup the client's topics and trigger the metadata deferred
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
ret.callback(None)
# Expect that only the msgs2 messages were sent
msgSet = create_message_set(
make_send_requests(msgs2), producer.codec)
req = ProduceRequest(self.topic, 1, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=producer.ack_timeout,
fail_on_error=False)
producer.stop()
def test_producer_send_messages_no_acks(self):
first_part = 19
client = Mock()
ret = Deferred()
client.send_produce_request.return_value = ret
client.topic_partitions = {self.topic: [first_part, 101, 102, 103]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
ack_timeout = 5
producer = Producer(client, ack_timeout=ack_timeout,
req_acks=PRODUCER_ACK_NOT_REQUIRED)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, first_part, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=ack_timeout,
fail_on_error=False)
# Check results when "response" fires
self.assertNoResult(d)
ret.callback([])
result = self.successResultOf(d)
self.assertEqual(result, None)
producer.stop()
def test_producer_send_messages_batched(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f), succeed([ProduceResponse(self.topic, 0, 0, 10L)])]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client,
batch_every_n=batch_n,
batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, ANY, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
self.successResultOf(d)
producer.stop()
def test_producer_send_messages_batched(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f), succeed([ProduceResponse(self.topic, 0, 0, 10L)])]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client, batch_every_n=batch_n, batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, ANY, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=producer.ack_timeout,
fail_on_error=False)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
self.successResultOf(d)
producer.stop()
def test_producer_send_messages(self):
first_part = 23
client = Mock()
ret = Deferred()
client.send_produce_request.return_value = ret
client.topic_partitions = {self.topic: [first_part, 101, 102, 103]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
ack_timeout = 5
producer = Producer(client, ack_timeout=ack_timeout)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, first_part, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=ack_timeout,
fail_on_error=False)
# Check results when "response" fires
self.assertNoResult(d)
resp = [ProduceResponse(self.topic, first_part, 0, 10L)]
ret.callback(resp)
result = self.successResultOf(d)
self.assertEqual(result, resp[0])
producer.stop()
def test_producer_cancel_one_request_getting_topic(self):
# Test cancelling a request after it's begun to be processed
client = Mock()
client.topic_partitions = {}
ret = Deferred()
client.load_metadata_for_topics.return_value = ret
msgs = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
batch_n = 4
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
# This will trigger the metadata lookup
d2 = producer.send_messages(self.topic, msgs=msgs2)
d1.cancel()
self.failureResultOf(d1, CancelledError)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
self.assertNoResult(d2)
# Setup the client's topics and trigger the metadata deferred
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
ret.callback(None)
# Expect that only the msgs2 messages were sent
msgSet = create_message_set(make_send_requests(msgs2), producer.codec)
req = ProduceRequest(self.topic, 1, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
producer.stop()
def test_producer_send_messages_batched_fail(self):
client = Mock()
ret = [Deferred(), Deferred(), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
batch_t = 5
clock = MemoryReactorClock()
producer = Producer(client, batch_every_t=batch_t, batch_send=True,
clock=clock, max_req_attempts=3)
# Advance the clock to ensure when no messages to send no error
clock.advance(batch_t)
d = producer.send_messages(self.topic, msgs=msgs)
# Check no request was yet sent
self.assertFalse(client.send_produce_request.called)
# Advance the clock
clock.advance(batch_t)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
produce_request_call = call([req], acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
produce_request_calls = [produce_request_call]
client.send_produce_request.assert_has_calls(produce_request_calls)
self.assertNoResult(d)
# Fire the failure from the first request to the client
ret[0].errback(OffsetOutOfRangeError(
'test_producer_send_messages_batched_fail'))
# Still no result, producer should retry first
self.assertNoResult(d)
# Check retry wasn't immediate
self.assertEqual(client.send_produce_request.call_count, 1)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval)
# Check 2nd send_produce_request (1st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[1].errback(BrokerNotAvailableError(
'test_producer_send_messages_batched_fail_2'))
# Still no result, producer should retry one more time
self.assertNoResult(d)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval * 1.1)
# Check 3nd send_produce_request (2st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[2].errback(LeaderNotAvailableError(
'test_producer_send_messages_batched_fail_3'))
self.failureResultOf(d, LeaderNotAvailableError)
producer.stop()
def test_producer_send_messages_no_retry_fail(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
client.send_produce_request.side_effect = [fail(f)]
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client, max_req_attempts=1)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=producer.ack_timeout,
fail_on_error=False)
self.failureResultOf(d, BrokerNotAvailableError)
producer.stop()
def test_producer_send_messages_no_retry_fail(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
client.send_produce_request.side_effect = [fail(f)]
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client, max_req_attempts=1)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
self.failureResultOf(d, BrokerNotAvailableError)
producer.stop()
def test_producer_send_messages_batched_fail(self):
client = Mock()
ret = [Deferred(), Deferred(), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
batch_t = 5
clock = MemoryReactorClock()
producer = Producer(client,
batch_every_t=batch_t,
batch_send=True,
clock=clock,
max_req_attempts=3)
# Advance the clock to ensure when no messages to send no error
clock.advance(batch_t)
d = producer.send_messages(self.topic, msgs=msgs)
# Check no request was yet sent
self.assertFalse(client.send_produce_request.called)
# Advance the clock
clock.advance(batch_t)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
produce_request_call = call([req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
produce_request_calls = [produce_request_call]
client.send_produce_request.assert_has_calls(produce_request_calls)
self.assertNoResult(d)
# Fire the failure from the first request to the client
ret[0].errback(
OffsetOutOfRangeError('test_producer_send_messages_batched_fail'))
# Still no result, producer should retry first
self.assertNoResult(d)
# Check retry wasn't immediate
self.assertEqual(client.send_produce_request.call_count, 1)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval)
# Check 2nd send_produce_request (1st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[1].errback(
BrokerNotAvailableError(
'test_producer_send_messages_batched_fail_2'))
# Still no result, producer should retry one more time
self.assertNoResult(d)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval * 1.1)
# Check 3nd send_produce_request (2st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[2].errback(
LeaderNotAvailableError(
'test_producer_send_messages_batched_fail_3'))
self.failureResultOf(d, LeaderNotAvailableError)
producer.stop()
