def test_producer_send_messages_keyed_same_partition(self):
"""test_producer_send_messages_keyed_same_partition
Test that messages sent with a key are actually sent with that key,
even if they go to the same topic/partition (batching preserves keys)
"""
first_part = 43
second_part = 55
client = Mock()
ret1 = Deferred()
client.send_produce_request.side_effect = [ret1]
client.topic_partitions = {self.topic: [first_part, second_part]}
client.metadata_error_for_topic.return_value = False
msgs1 = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("odd_man_out")]
msgs3 = [self.msg("three"), self.msg("four")]
key1 = '99'
key3 = 'foo'
ack_timeout = 5
# Even though we're sending keyed messages, we use the default
# round-robin partitioner, since the requests are easier to predict
producer = Producer(client,
ack_timeout=ack_timeout,
batch_send=True,
batch_every_n=4)
d1 = producer.send_messages(self.topic, key=key1, msgs=msgs1)
d2 = producer.send_messages(self.topic, msgs=msgs2)
d3 = producer.send_messages(self.topic, key=key3, msgs=msgs3)
# Check the expected request was sent
msgSet1 = create_message_set([
make_send_requests(msgs1, key=key1)[0],
make_send_requests(msgs3, key=key3)[0]
], producer.codec)
msgSet2 = create_message_set(make_send_requests(msgs2), producer.codec)
req1 = ProduceRequest(self.topic, first_part, msgSet1)
req2 = ProduceRequest(self.topic, second_part, msgSet2)
# Annoying, but order of requests is indeterminate...
client.send_produce_request.assert_called_once_with(
ANY,
acks=producer.req_acks,
timeout=ack_timeout,
fail_on_error=False)
self.assertEqual(sorted([req1, req2]),
sorted(client.send_produce_request.call_args[0][0]))
# Check results when "response" fires
self.assertNoResult(d1)
self.assertNoResult(d2)
self.assertNoResult(d3)
resp = [
ProduceResponse(self.topic, first_part, 0, 10L),
ProduceResponse(self.topic, second_part, 0, 23L)
]
ret1.callback(resp)
result = self.successResultOf(d1)
self.assertEqual(result, resp[0])
result = self.successResultOf(d2)
self.assertEqual(result, resp[1])
result = self.successResultOf(d3)
self.assertEqual(result, resp[0])
producer.stop()
def test_create_message_set(self):
messages = [b'1', b'2', b'3']
reqs = make_send_requests(messages)
# Default codec is CODEC_NONE. Expect list of regular messages.
expect = [sentinel.message] * len(messages)
with self.mock_create_message_fns():
message_set = create_message_set(reqs)
self.assertEqual(message_set, expect)
# CODEC_NONE: Expect list of regular messages.
expect = [sentinel.message] * len(messages)
with self.mock_create_message_fns():
message_set = create_message_set(reqs, CODEC_NONE)
self.assertEqual(message_set, expect)
# CODEC_GZIP: Expect list of one gzip-encoded message.
expect = [sentinel.gzip_message]
with self.mock_create_message_fns():
message_set = create_message_set(reqs, CODEC_GZIP)
self.assertEqual(message_set, expect)
# CODEC_SNAPPY: Expect list of one snappy-encoded message.
expect = [sentinel.snappy_message]
with self.mock_create_message_fns():
message_set = create_message_set(reqs, CODEC_SNAPPY)
self.assertEqual(message_set, expect)
# Unknown codec should raise UnsupportedCodecError.
self.assertRaises(UnsupportedCodecError,
create_message_set, reqs, -1)
def test_create_message_set(self):
messages = [b'1', b'2', b'3']
reqs = make_send_requests(messages)
# Default codec is CODEC_NONE. Expect list of regular messages.
expect = [sentinel.message] * len(messages)
with self.mock_create_message_fns():
message_set = create_message_set(reqs)
self.assertEqual(message_set, expect)
# CODEC_NONE: Expect list of regular messages.
expect = [sentinel.message] * len(messages)
with self.mock_create_message_fns():
message_set = create_message_set(reqs, CODEC_NONE)
self.assertEqual(message_set, expect)
# CODEC_GZIP: Expect list of one gzip-encoded message.
expect = [sentinel.gzip_message]
with self.mock_create_message_fns():
message_set = create_message_set(reqs, CODEC_GZIP)
self.assertEqual(message_set, expect)
# CODEC_SNAPPY: Expect list of one snappy-encoded message.
expect = [sentinel.snappy_message]
with self.mock_create_message_fns():
message_set = create_message_set(reqs, CODEC_SNAPPY)
self.assertEqual(message_set, expect)
# Unknown codec should raise UnsupportedCodecError.
self.assertRaises(UnsupportedCodecError, create_message_set, reqs, -1)
def test_producer_send_messages_keyed_same_partition(self):
"""test_producer_send_messages_keyed_same_partition
Test that messages sent with a key are actually sent with that key,
even if they go to the same topic/partition (batching preserves keys)
"""
first_part = 43
second_part = 55
client = Mock()
ret1 = Deferred()
client.send_produce_request.side_effect = [ret1]
client.topic_partitions = {self.topic: [first_part, second_part]}
client.metadata_error_for_topic.return_value = False
msgs1 = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("odd_man_out")]
msgs3 = [self.msg("three"), self.msg("four")]
key1 = '99'
key3 = 'foo'
ack_timeout = 5
# Even though we're sending keyed messages, we use the default
# round-robin partitioner, since the requests are easier to predict
producer = Producer(client, ack_timeout=ack_timeout, batch_send=True,
batch_every_n=4)
d1 = producer.send_messages(self.topic, key=key1, msgs=msgs1)
d2 = producer.send_messages(self.topic, msgs=msgs2)
d3 = producer.send_messages(self.topic, key=key3, msgs=msgs3)
# Check the expected request was sent
msgSet1 = create_message_set(
[make_send_requests(msgs1, key=key1)[0],
make_send_requests(msgs3, key=key3)[0]], producer.codec)
msgSet2 = create_message_set(make_send_requests(
msgs2), producer.codec)
req1 = ProduceRequest(self.topic, first_part, msgSet1)
req2 = ProduceRequest(self.topic, second_part, msgSet2)
# Annoying, but order of requests is indeterminate...
client.send_produce_request.assert_called_once_with(
ANY, acks=producer.req_acks, timeout=ack_timeout,
fail_on_error=False)
self.assertEqual(sorted([req1, req2]),
sorted(client.send_produce_request.call_args[0][0]))
# Check results when "response" fires
self.assertNoResult(d1)
self.assertNoResult(d2)
self.assertNoResult(d3)
resp = [ProduceResponse(self.topic, first_part, 0, 10L),
ProduceResponse(self.topic, second_part, 0, 23L)]
ret1.callback(resp)
result = self.successResultOf(d1)
self.assertEqual(result, resp[0])
result = self.successResultOf(d2)
self.assertEqual(result, resp[1])
result = self.successResultOf(d3)
self.assertEqual(result, resp[0])
producer.stop()
def test_producer_cancel_one_request_getting_topic(self):
# Test cancelling a request after it's begun to be processed
client = Mock()
client.topic_partitions = {}
ret = Deferred()
client.load_metadata_for_topics.return_value = ret
msgs = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
batch_n = 4
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
# This will trigger the metadata lookup
d2 = producer.send_messages(self.topic, msgs=msgs2)
d1.cancel()
self.failureResultOf(d1, CancelledError)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
self.assertNoResult(d2)
# Setup the client's topics and trigger the metadata deferred
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
ret.callback(None)
# Expect that only the msgs2 messages were sent
msgSet = create_message_set(
make_send_requests(msgs2), producer.codec)
req = ProduceRequest(self.topic, 1, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=producer.ack_timeout,
fail_on_error=False)
producer.stop()
def test_producer_send_messages_batched(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f), succeed([ProduceResponse(self.topic, 0, 0, 10L)])]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client, batch_every_n=batch_n, batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, ANY, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=producer.ack_timeout,
fail_on_error=False)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
self.successResultOf(d)
producer.stop()
def test_producer_send_messages_no_acks(self):
first_part = 19
client = Mock()
ret = Deferred()
client.send_produce_request.return_value = ret
client.topic_partitions = {self.topic: [first_part, 101, 102, 103]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
ack_timeout = 5
producer = Producer(client, ack_timeout=ack_timeout,
req_acks=PRODUCER_ACK_NOT_REQUIRED)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, first_part, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=ack_timeout,
fail_on_error=False)
# Check results when "response" fires
self.assertNoResult(d)
ret.callback([])
result = self.successResultOf(d)
self.assertEqual(result, None)
producer.stop()
def test_producer_send_messages(self):
first_part = 23
client = Mock()
ret = Deferred()
client.send_produce_request.return_value = ret
client.topic_partitions = {self.topic: [first_part, 101, 102, 103]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
ack_timeout = 5
producer = Producer(client, ack_timeout=ack_timeout)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, first_part, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=ack_timeout,
fail_on_error=False)
# Check results when "response" fires
self.assertNoResult(d)
resp = [ProduceResponse(self.topic, first_part, 0, 10L)]
ret.callback(resp)
result = self.successResultOf(d)
self.assertEqual(result, resp[0])
producer.stop()
def test_producer_cancel_one_request_getting_topic(self):
# Test cancelling a request after it's begun to be processed
client = Mock()
client.topic_partitions = {}
ret = Deferred()
client.load_metadata_for_topics.return_value = ret
msgs = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
batch_n = 4
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
# This will trigger the metadata lookup
d2 = producer.send_messages(self.topic, msgs=msgs2)
d1.cancel()
self.failureResultOf(d1, CancelledError)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
self.assertNoResult(d2)
# Setup the client's topics and trigger the metadata deferred
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
ret.callback(None)
# Expect that only the msgs2 messages were sent
msgSet = create_message_set(make_send_requests(msgs2), producer.codec)
req = ProduceRequest(self.topic, 1, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
producer.stop()
def test_producer_send_messages_batched(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f), succeed([ProduceResponse(self.topic, 0, 0, 10L)])]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client,
batch_every_n=batch_n,
batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, ANY, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
self.successResultOf(d)
producer.stop()
def test_producer_send_messages_batched_fail(self):
client = Mock()
ret = [Deferred(), Deferred(), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
batch_t = 5
clock = MemoryReactorClock()
producer = Producer(client, batch_every_t=batch_t, batch_send=True,
clock=clock, max_req_attempts=3)
# Advance the clock to ensure when no messages to send no error
clock.advance(batch_t)
d = producer.send_messages(self.topic, msgs=msgs)
# Check no request was yet sent
self.assertFalse(client.send_produce_request.called)
# Advance the clock
clock.advance(batch_t)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
produce_request_call = call([req], acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
produce_request_calls = [produce_request_call]
client.send_produce_request.assert_has_calls(produce_request_calls)
self.assertNoResult(d)
# Fire the failure from the first request to the client
ret[0].errback(OffsetOutOfRangeError(
'test_producer_send_messages_batched_fail'))
# Still no result, producer should retry first
self.assertNoResult(d)
# Check retry wasn't immediate
self.assertEqual(client.send_produce_request.call_count, 1)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval)
# Check 2nd send_produce_request (1st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[1].errback(BrokerNotAvailableError(
'test_producer_send_messages_batched_fail_2'))
# Still no result, producer should retry one more time
self.assertNoResult(d)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval * 1.1)
# Check 3nd send_produce_request (2st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[2].errback(LeaderNotAvailableError(
'test_producer_send_messages_batched_fail_3'))
self.failureResultOf(d, LeaderNotAvailableError)
producer.stop()
def test_producer_send_messages_no_retry_fail(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
client.send_produce_request.side_effect = [fail(f)]
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client, max_req_attempts=1)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=producer.ack_timeout,
fail_on_error=False)
self.failureResultOf(d, BrokerNotAvailableError)
producer.stop()
def test_producer_send_messages_no_retry_fail(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
client.send_produce_request.side_effect = [fail(f)]
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client, max_req_attempts=1)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
self.failureResultOf(d, BrokerNotAvailableError)
producer.stop()
def test_producer_send_messages_batched_fail(self):
client = Mock()
ret = [Deferred(), Deferred(), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
batch_t = 5
clock = MemoryReactorClock()
producer = Producer(client,
batch_every_t=batch_t,
batch_send=True,
clock=clock,
max_req_attempts=3)
# Advance the clock to ensure when no messages to send no error
clock.advance(batch_t)
d = producer.send_messages(self.topic, msgs=msgs)
# Check no request was yet sent
self.assertFalse(client.send_produce_request.called)
# Advance the clock
clock.advance(batch_t)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
produce_request_call = call([req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
produce_request_calls = [produce_request_call]
client.send_produce_request.assert_has_calls(produce_request_calls)
self.assertNoResult(d)
# Fire the failure from the first request to the client
ret[0].errback(
OffsetOutOfRangeError('test_producer_send_messages_batched_fail'))
# Still no result, producer should retry first
self.assertNoResult(d)
# Check retry wasn't immediate
self.assertEqual(client.send_produce_request.call_count, 1)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval)
# Check 2nd send_produce_request (1st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[1].errback(
BrokerNotAvailableError(
'test_producer_send_messages_batched_fail_2'))
# Still no result, producer should retry one more time
self.assertNoResult(d)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval * 1.1)
# Check 3nd send_produce_request (2st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[2].errback(
LeaderNotAvailableError(
'test_producer_send_messages_batched_fail_3'))
self.failureResultOf(d, LeaderNotAvailableError)
producer.stop()
