def setUp(self): self.client = MagicMock() self.queue = queue.Queue() self.registrar = EventRegistrar()
class TestKafkaProducerSendUpstream(unittest.TestCase): def setUp(self): self.client = MagicMock() self.queue = queue.Queue() self.registrar = EventRegistrar() def _run_process(self, retries_limit=3, sleep_timeout=1): # run _send_upstream process with the queue stop_event = threading.Event() retry_options = RetryOptions(limit=retries_limit, backoff_ms=50, retry_on_timeouts=False) self.thread = threading.Thread( target=_send_upstream, args=( self.queue, self.client, CODEC_NONE, 0.3, # batch time (seconds) 3, # batch length Producer.ACK_AFTER_LOCAL_WRITE, Producer.DEFAULT_ACK_TIMEOUT, retry_options, stop_event, self.registrar, ), ) self.thread.daemon = True self.thread.start() time.sleep(sleep_timeout) stop_event.set() def test_wo_retries(self): # lets create a queue and add 10 messages for 1 partition for i in range(10): self.queue.put((TopicPartition("test", 0), "msg %i", "key %i")) event_mock = Mock() self.registrar.register("async.producer.connect.succeed", event_mock) self._run_process() self.assertTrue(event_mock.called_once_with()) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 4 non-void cals: # 3 batches of 3 msgs each + 1 batch of 1 message self.assertEqual(self.client.send_produce_request.call_count, 4) def test_first_send_failed(self): # lets create a queue and add 10 messages for 10 different partitions # to show how retries should work ideally for i in range(10): self.queue.put((TopicPartition("test", i), "msg %i", "key %i")) # Mock offsets counter for closure offsets = collections.defaultdict(lambda: collections.defaultdict(lambda: 0)) self.client.is_first_time = True def send_side_effect(reqs, *args, **kwargs): if self.client.is_first_time: self.client.is_first_time = False return [FailedPayloadsError(req) for req in reqs] responses = [] for req in reqs: offset = offsets[req.topic][req.partition] offsets[req.topic][req.partition] += len(req.messages) responses.append(ProduceResponsePayload(req.topic, req.partition, 0, offset)) return responses self.client.send_produce_request.side_effect = send_side_effect self._run_process(2) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 5 non-void calls: 1st failed batch of 3 msgs # plus 3 batches of 3 msgs each + 1 batch of 1 message self.assertEqual(self.client.send_produce_request.call_count, 5) def test_with_limited_retries(self): # lets create a queue and add 10 messages for 10 different partitions # to show how retries should work ideally for i in range(10): self.queue.put((TopicPartition("test", i), "msg %i" % i, "key %i" % i)) def send_side_effect(reqs, *args, **kwargs): return [FailedPayloadsError(req) for req in reqs] self.client.send_produce_request.side_effect = send_side_effect self._run_process(3, 3) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 16 non-void calls: # 3 initial batches of 3 msgs each + 1 initial batch of 1 msg + # 3 retries of the batches above = (1 + 3 retries) * 4 batches = 16 self.assertEqual(self.client.send_produce_request.call_count, 16) def test_async_producer_not_leader(self): for i in range(10): self.queue.put((TopicPartition("test", i), "msg %i", "key %i")) # Mock offsets counter for closure offsets = collections.defaultdict(lambda: collections.defaultdict(lambda: 0)) self.client.is_first_time = True def send_side_effect(reqs, *args, **kwargs): if self.client.is_first_time: self.client.is_first_time = False return [ ProduceResponsePayload(req.topic, req.partition, NotLeaderForPartitionError.errno, -1) for req in reqs ] responses = [] for req in reqs: offset = offsets[req.topic][req.partition] offsets[req.topic][req.partition] += len(req.messages) responses.append(ProduceResponsePayload(req.topic, req.partition, 0, offset)) return responses self.client.send_produce_request.side_effect = send_side_effect self._run_process(2) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 5 non-void calls: 1st failed batch of 3 msgs # + 3 batches of 3 msgs each + 1 batch of 1 msg = 1 + 3 + 1 = 5 self.assertEqual(self.client.send_produce_request.call_count, 5) def tearDown(self): for _ in xrange(self.queue.qsize()): self.queue.get()
class TestKafkaProducerSendUpstream(unittest.TestCase): def setUp(self): self.client = MagicMock() self.queue = queue.Queue() self.registrar = EventRegistrar() def _run_process(self, retries_limit=3, sleep_timeout=1): # run _send_upstream process with the queue stop_event = threading.Event() retry_options = RetryOptions(limit=retries_limit, backoff_ms=50, retry_on_timeouts=False) self.thread = threading.Thread( target=_send_upstream, args=( self.queue, self.client, CODEC_NONE, 0.3, # batch time (seconds) 3, # batch length Producer.ACK_AFTER_LOCAL_WRITE, Producer.DEFAULT_ACK_TIMEOUT, retry_options, stop_event, self.registrar)) self.thread.daemon = True self.thread.start() time.sleep(sleep_timeout) stop_event.set() def test_wo_retries(self): # lets create a queue and add 10 messages for 1 partition for i in range(10): self.queue.put((TopicPartition("test", 0), "msg %i", "key %i")) event_mock = Mock() self.registrar.register('async.producer.connect.succeed', event_mock) self._run_process() self.assertTrue(event_mock.called_once_with()) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 4 non-void cals: # 3 batches of 3 msgs each + 1 batch of 1 message self.assertEqual(self.client.send_produce_request.call_count, 4) def test_first_send_failed(self): # lets create a queue and add 10 messages for 10 different partitions # to show how retries should work ideally for i in range(10): self.queue.put((TopicPartition("test", i), "msg %i", "key %i")) # Mock offsets counter for closure offsets = collections.defaultdict( lambda: collections.defaultdict(lambda: 0)) self.client.is_first_time = True def send_side_effect(reqs, *args, **kwargs): if self.client.is_first_time: self.client.is_first_time = False return [FailedPayloadsError(req) for req in reqs] responses = [] for req in reqs: offset = offsets[req.topic][req.partition] offsets[req.topic][req.partition] += len(req.messages) responses.append( ProduceResponsePayload(req.topic, req.partition, 0, offset)) return responses self.client.send_produce_request.side_effect = send_side_effect self._run_process(2) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 5 non-void calls: 1st failed batch of 3 msgs # plus 3 batches of 3 msgs each + 1 batch of 1 message self.assertEqual(self.client.send_produce_request.call_count, 5) def test_with_limited_retries(self): # lets create a queue and add 10 messages for 10 different partitions # to show how retries should work ideally for i in range(10): self.queue.put((TopicPartition("test", i), "msg %i" % i, "key %i" % i)) def send_side_effect(reqs, *args, **kwargs): return [FailedPayloadsError(req) for req in reqs] self.client.send_produce_request.side_effect = send_side_effect self._run_process(3, 3) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 16 non-void calls: # 3 initial batches of 3 msgs each + 1 initial batch of 1 msg + # 3 retries of the batches above = (1 + 3 retries) * 4 batches = 16 self.assertEqual(self.client.send_produce_request.call_count, 16) def test_async_producer_not_leader(self): for i in range(10): self.queue.put((TopicPartition("test", i), "msg %i", "key %i")) # Mock offsets counter for closure offsets = collections.defaultdict( lambda: collections.defaultdict(lambda: 0)) self.client.is_first_time = True def send_side_effect(reqs, *args, **kwargs): if self.client.is_first_time: self.client.is_first_time = False return [ ProduceResponsePayload(req.topic, req.partition, NotLeaderForPartitionError.errno, -1) for req in reqs ] responses = [] for req in reqs: offset = offsets[req.topic][req.partition] offsets[req.topic][req.partition] += len(req.messages) responses.append( ProduceResponsePayload(req.topic, req.partition, 0, offset)) return responses self.client.send_produce_request.side_effect = send_side_effect self._run_process(2) # the queue should be void at the end of the test self.assertEqual(self.queue.empty(), True) # there should be 5 non-void calls: 1st failed batch of 3 msgs # + 3 batches of 3 msgs each + 1 batch of 1 msg = 1 + 3 + 1 = 5 self.assertEqual(self.client.send_produce_request.call_count, 5) def tearDown(self): for _ in xrange(self.queue.qsize()): self.queue.get()