def test_producer_send_messages_batched(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f), succeed([ProduceResponse(self.topic, 0, 0, 10L)])]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client,
batch_every_n=batch_n,
batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, ANY, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
self.successResultOf(d)
producer.stop()
def test_producer_send_messages_no_retry_fail(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
client.send_produce_request.side_effect = [fail(f)]
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client, max_req_attempts=1)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
self.failureResultOf(d, BrokerNotAvailableError)
producer.stop()
def test_producer_stop_during_request(self):
"""
Test stopping producer while it's waiting for reply from client
"""
clock = MemoryReactorClock()
client = Mock(reactor=clock)
f = Failure(BrokerNotAvailableError())
ret = [fail(f), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
batch_n = 2
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d = producer.send_messages(self.topic, msgs=msgs)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
producer.stop()
self.failureResultOf(d, tid_CancelledError)
def test_producer_stop_waiting_to_retry(self):
# Test stopping producer while it's waiting to retry a request
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f)]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client,
batch_every_n=batch_n,
batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
# Advance the clock, some, but not enough to retry
clock.advance(producer._retry_interval / 2)
# Stop the producer before the retry
producer.stop()
self.failureResultOf(d, tid_CancelledError)
def test_producer_send_messages_batched_fail(self):
client = Mock()
ret = [Deferred(), Deferred(), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
batch_t = 5
clock = MemoryReactorClock()
producer = Producer(client,
batch_every_t=batch_t,
batch_send=True,
clock=clock,
max_req_attempts=3)
# Advance the clock to ensure when no messages to send no error
clock.advance(batch_t)
d = producer.send_messages(self.topic, msgs=msgs)
# Check no request was yet sent
self.assertFalse(client.send_produce_request.called)
# Advance the clock
clock.advance(batch_t)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
produce_request_call = call([req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
produce_request_calls = [produce_request_call]
client.send_produce_request.assert_has_calls(produce_request_calls)
self.assertNoResult(d)
# Fire the failure from the first request to the client
ret[0].errback(
OffsetOutOfRangeError('test_producer_send_messages_batched_fail'))
# Still no result, producer should retry first
self.assertNoResult(d)
# Check retry wasn't immediate
self.assertEqual(client.send_produce_request.call_count, 1)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval)
# Check 2nd send_produce_request (1st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[1].errback(
BrokerNotAvailableError(
'test_producer_send_messages_batched_fail_2'))
# Still no result, producer should retry one more time
self.assertNoResult(d)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval * 1.1)
# Check 3nd send_produce_request (2st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[2].errback(
LeaderNotAvailableError(
'test_producer_send_messages_batched_fail_3'))
self.failureResultOf(d, LeaderNotAvailableError)
producer.stop()
def test_producer_send_messages_batched_partial_success(self):
"""test_producer_send_messages_batched_partial_success
This tests the complexity of the error handling for a single batch
request.
Scenario: The producer's caller sends 5 requests to two (total) topics
The client's metadata is such that the producer will produce
requests to post msgs to 5 separate topic/partition tuples
The batch size is reached, so the producer sends the request
The caller then cancels one of the requests
The (mock) client returns partial success in the form of a
FailedPayloadsError.
The Producer then should return the successful results and
retry the failed.
The (mock) client then "succeeds" the remaining results.
"""
client = Mock()
topic2 = 'tpsmbps_two'
client.topic_partitions = {self.topic: [0, 1, 2, 3], topic2: [4, 5, 6]}
client.metadata_error_for_topic.return_value = False
init_resp = [
ProduceResponse(self.topic, 0, 0, 10L),
ProduceResponse(self.topic, 1, 6, 20L),
ProduceResponse(topic2, 5, 0, 30L),
]
next_resp = [
ProduceResponse(self.topic, 2, 0, 10L),
ProduceResponse(self.topic, 1, 0, 20L),
ProduceResponse(topic2, 4, 0, 30L),
]
failed_payloads = [
(ProduceRequest(self.topic, ANY,
ANY), NotLeaderForPartitionError()),
(ProduceRequest(topic2, ANY, ANY), BrokerNotAvailableError()),
]
f = Failure(FailedPayloadsError(init_resp, failed_payloads))
ret = [fail(f), succeed(next_resp)]
client.send_produce_request.side_effect = ret
msgs = self.msgs(range(10))
results = []
clock = MemoryReactorClock()
producer = Producer(client,
batch_send=True,
batch_every_t=0,
clock=clock)
# Send 5 total requests: 4 here, one after we make sure we didn't
# send early
results.append(producer.send_messages(self.topic, msgs=msgs[0:3]))
results.append(producer.send_messages(topic2, msgs=msgs[3:5]))
results.append(producer.send_messages(self.topic, msgs=msgs[5:8]))
results.append(producer.send_messages(topic2, msgs=msgs[8:9]))
# No call yet, not enough messages
self.assertFalse(client.send_produce_request.called)
# Enough messages to start the request
results.append(producer.send_messages(self.topic, msgs=msgs[9:10]))
# Before the retry, there should be some results
self.assertEqual(init_resp[0], self.successResultOf(results[0]))
self.assertEqual(init_resp[2], self.successResultOf(results[3]))
# Advance the clock
clock.advance(producer._retry_interval)
# Check the otehr results came in
self.assertEqual(next_resp[0], self.successResultOf(results[4]))
self.assertEqual(next_resp[1], self.successResultOf(results[2]))
self.assertEqual(next_resp[2], self.successResultOf(results[1]))
producer.stop()
