def test_produce_mixed(self): start_offset = yield self.current_offset(self.topic, 0) msg_count = 1+100 messages = [ create_message("Just a plain message"), create_message_set( make_send_requests( ["Gzipped %d" % i for i in range(100)]), CODEC_GZIP)[0] ] # Can't produce snappy messages without snappy... if has_snappy(): msg_count += 100 messages.append( create_message_set( make_send_requests(["Snappy %d" % i for i in range(100)]), CODEC_SNAPPY)[0]) yield self.assert_produce_request(messages, start_offset, msg_count) ptMsgs = ['Just a plain message'] ptMsgs.extend(["Gzipped %d" % i for i in range(100)]) if has_snappy(): ptMsgs.extend(["Snappy %d" % i for i in range(100)]) yield self.assert_fetch_offset(0, start_offset, ptMsgs, fetch_size=10240)
def test_producer_send_messages_keyed_same_partition(self): """test_producer_send_messages_keyed_same_partition Test that messages sent with a key are actually sent with that key, even if they go to the same topic/partition (batching preserves keys) """ first_part = 43 second_part = 55 client = Mock() ret1 = Deferred() client.send_produce_request.side_effect = [ret1] client.topic_partitions = {self.topic: [first_part, second_part]} client.metadata_error_for_topic.return_value = False msgs1 = [self.msg("one"), self.msg("two")] msgs2 = [self.msg("odd_man_out")] msgs3 = [self.msg("three"), self.msg("four")] key1 = '99' key3 = 'foo' ack_timeout = 5 # Even though we're sending keyed messages, we use the default # round-robin partitioner, since the requests are easier to predict producer = Producer(client, ack_timeout=ack_timeout, batch_send=True, batch_every_n=4) d1 = producer.send_messages(self.topic, key=key1, msgs=msgs1) d2 = producer.send_messages(self.topic, msgs=msgs2) d3 = producer.send_messages(self.topic, key=key3, msgs=msgs3) # Check the expected request was sent msgSet1 = create_message_set([ make_send_requests(msgs1, key=key1)[0], make_send_requests(msgs3, key=key3)[0] ], producer.codec) msgSet2 = create_message_set(make_send_requests(msgs2), producer.codec) req1 = ProduceRequest(self.topic, first_part, msgSet1) req2 = ProduceRequest(self.topic, second_part, msgSet2) # Annoying, but order of requests is indeterminate... client.send_produce_request.assert_called_once_with( ANY, acks=producer.req_acks, timeout=ack_timeout, fail_on_error=False) self.assertEqual(sorted([req1, req2]), sorted(client.send_produce_request.call_args[0][0])) # Check results when "response" fires self.assertNoResult(d1) self.assertNoResult(d2) self.assertNoResult(d3) resp = [ ProduceResponse(self.topic, first_part, 0, 10L), ProduceResponse(self.topic, second_part, 0, 23L) ] ret1.callback(resp) result = self.successResultOf(d1) self.assertEqual(result, resp[0]) result = self.successResultOf(d2) self.assertEqual(result, resp[1]) result = self.successResultOf(d3) self.assertEqual(result, resp[0]) producer.stop()
def test_produce_mixed(self): start_offset = yield self.current_offset(self.topic, 0) msg_count = 1 + 100 messages = [ create_message("Just a plain message"), create_message_set( make_send_requests(["Gzipped %d" % i for i in range(100)]), CODEC_GZIP)[0] ] # Can't produce snappy messages without snappy... if has_snappy(): msg_count += 100 messages.append( create_message_set( make_send_requests(["Snappy %d" % i for i in range(100)]), CODEC_SNAPPY)[0]) yield self.assert_produce_request(messages, start_offset, msg_count) ptMsgs = ['Just a plain message'] ptMsgs.extend(["Gzipped %d" % i for i in range(100)]) if has_snappy(): ptMsgs.extend(["Snappy %d" % i for i in range(100)]) yield self.assert_fetch_offset(0, start_offset, ptMsgs, fetch_size=10240)
def test_producer_send_messages_keyed_same_partition(self): """test_producer_send_messages_keyed_same_partition Test that messages sent with a key are actually sent with that key, even if they go to the same topic/partition (batching preserves keys) """ first_part = 43 second_part = 55 client = Mock() ret1 = Deferred() client.send_produce_request.side_effect = [ret1] client.topic_partitions = {self.topic: [first_part, second_part]} client.metadata_error_for_topic.return_value = False msgs1 = [self.msg("one"), self.msg("two")] msgs2 = [self.msg("odd_man_out")] msgs3 = [self.msg("three"), self.msg("four")] key1 = '99' key3 = 'foo' ack_timeout = 5 # Even though we're sending keyed messages, we use the default # round-robin partitioner, since the requests are easier to predict producer = Producer(client, ack_timeout=ack_timeout, batch_send=True, batch_every_n=4) d1 = producer.send_messages(self.topic, key=key1, msgs=msgs1) d2 = producer.send_messages(self.topic, msgs=msgs2) d3 = producer.send_messages(self.topic, key=key3, msgs=msgs3) # Check the expected request was sent msgSet1 = create_message_set( [make_send_requests(msgs1, key=key1)[0], make_send_requests(msgs3, key=key3)[0]], producer.codec) msgSet2 = create_message_set(make_send_requests( msgs2), producer.codec) req1 = ProduceRequest(self.topic, first_part, msgSet1) req2 = ProduceRequest(self.topic, second_part, msgSet2) # Annoying, but order of requests is indeterminate... client.send_produce_request.assert_called_once_with( ANY, acks=producer.req_acks, timeout=ack_timeout, fail_on_error=False) self.assertEqual(sorted([req1, req2]), sorted(client.send_produce_request.call_args[0][0])) # Check results when "response" fires self.assertNoResult(d1) self.assertNoResult(d2) self.assertNoResult(d3) resp = [ProduceResponse(self.topic, first_part, 0, 10L), ProduceResponse(self.topic, second_part, 0, 23L)] ret1.callback(resp) result = self.successResultOf(d1) self.assertEqual(result, resp[0]) result = self.successResultOf(d2) self.assertEqual(result, resp[1]) result = self.successResultOf(d3) self.assertEqual(result, resp[0]) producer.stop()
def test_produce_10k_gzipped(self): start_offset = yield self.current_offset(self.topic, 0) msgs = create_message_set( make_send_requests( ["Gzipped batch 1, message %d" % i for i in range(5000)]), CODEC_GZIP) yield self.assert_produce_request(msgs, start_offset, 5000) msgs = create_message_set( make_send_requests( ["Gzipped batch 2, message %d" % i for i in range(5000)]), CODEC_GZIP) yield self.assert_produce_request(msgs, start_offset + 5000, 5000)
def test_produce_10k_gzipped(self): start_offset = yield self.current_offset(self.topic, 0) msgs = create_message_set( make_send_requests( ["Gzipped batch 1, message %d" % i for i in range(5000)]), CODEC_GZIP) yield self.assert_produce_request(msgs, start_offset, 5000) msgs = create_message_set( make_send_requests( ["Gzipped batch 2, message %d" % i for i in range(5000)]), CODEC_GZIP) yield self.assert_produce_request(msgs, start_offset+5000, 5000)
def test_produce_many_snappy(self): start_offset = yield self.current_offset(self.topic, 0) message1 = create_message_set( make_send_requests(["Snappy 1 %d" % i for i in range(100)]), CODEC_SNAPPY)[0] message2 = create_message_set( make_send_requests(["Snappy 2 %d" % i for i in range(100)]), CODEC_SNAPPY)[0] yield self.assert_produce_request( [message1, message2], start_offset, 200, )
def test_produce_many_snappy(self): start_offset = yield self.current_offset(self.topic, 0) message1 = create_message_set( make_send_requests( ["Snappy 1 %d" % i for i in range(100)]), CODEC_SNAPPY)[0] message2 = create_message_set( make_send_requests( ["Snappy 2 %d" % i for i in range(100)]), CODEC_SNAPPY)[0] yield self.assert_produce_request( [message1, message2], start_offset, 200, )
def test_produce_many_gzip(self): start_offset = yield self.current_offset(self.topic, 0) message1 = create_message_set( make_send_requests(["Gzipped 1 %d" % i for i in range(100)]), CODEC_GZIP)[0] message2 = create_message_set( make_send_requests(["Gzipped 2 %d" % i for i in range(100)]), CODEC_GZIP)[0] yield self.assert_produce_request( [message1, message2], start_offset, 200, )
def test_produce_many_gzip(self): start_offset = yield self.current_offset(self.topic, 0) message1 = create_message_set( make_send_requests( ["Gzipped 1 %d" % i for i in range(100)]), CODEC_GZIP)[0] message2 = create_message_set( make_send_requests( ["Gzipped 2 %d" % i for i in range(100)]), CODEC_GZIP)[0] yield self.assert_produce_request( [message1, message2], start_offset, 200, )
def test_producer_cancel_one_request_getting_topic(self): # Test cancelling a request after it's begun to be processed client = Mock() client.topic_partitions = {} ret = Deferred() client.load_metadata_for_topics.return_value = ret msgs = [self.msg("one"), self.msg("two")] msgs2 = [self.msg("three"), self.msg("four")] batch_n = 4 producer = Producer(client, batch_every_n=batch_n, batch_send=True) d1 = producer.send_messages(self.topic, msgs=msgs) # Check that no request was sent self.assertFalse(client.send_produce_request.called) # This will trigger the metadata lookup d2 = producer.send_messages(self.topic, msgs=msgs2) d1.cancel() self.failureResultOf(d1, CancelledError) # Check that still no request was sent self.assertFalse(client.send_produce_request.called) self.assertNoResult(d2) # Setup the client's topics and trigger the metadata deferred client.topic_partitions = {self.topic: [0, 1, 2, 3]} client.metadata_error_for_topic.return_value = False ret.callback(None) # Expect that only the msgs2 messages were sent msgSet = create_message_set( make_send_requests(msgs2), producer.codec) req = ProduceRequest(self.topic, 1, msgSet) client.send_produce_request.assert_called_once_with( [req], acks=producer.req_acks, timeout=producer.ack_timeout, fail_on_error=False) producer.stop()
def test_producer_send_messages_no_acks(self): first_part = 19 client = Mock() ret = Deferred() client.send_produce_request.return_value = ret client.topic_partitions = {self.topic: [first_part, 101, 102, 103]} client.metadata_error_for_topic.return_value = False msgs = [self.msg("one"), self.msg("two")] ack_timeout = 5 producer = Producer(client, ack_timeout=ack_timeout, req_acks=PRODUCER_ACK_NOT_REQUIRED) d = producer.send_messages(self.topic, msgs=msgs) # Check the expected request was sent msgSet = create_message_set( make_send_requests(msgs), producer.codec) req = ProduceRequest(self.topic, first_part, msgSet) client.send_produce_request.assert_called_once_with( [req], acks=producer.req_acks, timeout=ack_timeout, fail_on_error=False) # Check results when "response" fires self.assertNoResult(d) ret.callback([]) result = self.successResultOf(d) self.assertEqual(result, None) producer.stop()
def test_producer_send_messages_batched(self): client = Mock() f = Failure(BrokerNotAvailableError()) ret = [fail(f), succeed([ProduceResponse(self.topic, 0, 0, 10L)])] client.send_produce_request.side_effect = ret client.topic_partitions = {self.topic: [0, 1, 2, 3]} client.metadata_error_for_topic.return_value = False msgs = [self.msg("one"), self.msg("two")] clock = MemoryReactorClock() batch_n = 2 producer = Producer(client, batch_every_n=batch_n, batch_send=True, clock=clock) d = producer.send_messages(self.topic, msgs=msgs) # Check the expected request was sent msgSet = create_message_set(make_send_requests(msgs), producer.codec) req = ProduceRequest(self.topic, ANY, msgSet) client.send_produce_request.assert_called_once_with( [req], acks=producer.req_acks, timeout=producer.ack_timeout, fail_on_error=False) # At first, there's no result. Have to retry due to first failure self.assertNoResult(d) clock.advance(producer._retry_interval) self.successResultOf(d) producer.stop()
def test_producer_send_messages_batched(self): client = Mock() f = Failure(BrokerNotAvailableError()) ret = [fail(f), succeed([ProduceResponse(self.topic, 0, 0, 10L)])] client.send_produce_request.side_effect = ret client.topic_partitions = {self.topic: [0, 1, 2, 3]} client.metadata_error_for_topic.return_value = False msgs = [self.msg("one"), self.msg("two")] clock = MemoryReactorClock() batch_n = 2 producer = Producer(client, batch_every_n=batch_n, batch_send=True, clock=clock) d = producer.send_messages(self.topic, msgs=msgs) # Check the expected request was sent msgSet = create_message_set( make_send_requests(msgs), producer.codec) req = ProduceRequest(self.topic, ANY, msgSet) client.send_produce_request.assert_called_once_with( [req], acks=producer.req_acks, timeout=producer.ack_timeout, fail_on_error=False) # At first, there's no result. Have to retry due to first failure self.assertNoResult(d) clock.advance(producer._retry_interval) self.successResultOf(d) producer.stop()
def test_producer_send_messages(self): first_part = 23 client = Mock() ret = Deferred() client.send_produce_request.return_value = ret client.topic_partitions = {self.topic: [first_part, 101, 102, 103]} client.metadata_error_for_topic.return_value = False msgs = [self.msg("one"), self.msg("two")] ack_timeout = 5 producer = Producer(client, ack_timeout=ack_timeout) d = producer.send_messages(self.topic, msgs=msgs) # Check the expected request was sent msgSet = create_message_set(make_send_requests(msgs), producer.codec) req = ProduceRequest(self.topic, first_part, msgSet) client.send_produce_request.assert_called_once_with( [req], acks=producer.req_acks, timeout=ack_timeout, fail_on_error=False) # Check results when "response" fires self.assertNoResult(d) resp = [ProduceResponse(self.topic, first_part, 0, 10L)] ret.callback(resp) result = self.successResultOf(d) self.assertEqual(result, resp[0]) producer.stop()
def test_producer_cancel_one_request_getting_topic(self): # Test cancelling a request after it's begun to be processed client = Mock() client.topic_partitions = {} ret = Deferred() client.load_metadata_for_topics.return_value = ret msgs = [self.msg("one"), self.msg("two")] msgs2 = [self.msg("three"), self.msg("four")] batch_n = 4 producer = Producer(client, batch_every_n=batch_n, batch_send=True) d1 = producer.send_messages(self.topic, msgs=msgs) # Check that no request was sent self.assertFalse(client.send_produce_request.called) # This will trigger the metadata lookup d2 = producer.send_messages(self.topic, msgs=msgs2) d1.cancel() self.failureResultOf(d1, CancelledError) # Check that still no request was sent self.assertFalse(client.send_produce_request.called) self.assertNoResult(d2) # Setup the client's topics and trigger the metadata deferred client.topic_partitions = {self.topic: [0, 1, 2, 3]} client.metadata_error_for_topic.return_value = False ret.callback(None) # Expect that only the msgs2 messages were sent msgSet = create_message_set(make_send_requests(msgs2), producer.codec) req = ProduceRequest(self.topic, 1, msgSet) client.send_produce_request.assert_called_once_with( [req], acks=producer.req_acks, timeout=producer.ack_timeout, fail_on_error=False) producer.stop()
def test_producer_send_messages_batched_fail(self): client = Mock() ret = [Deferred(), Deferred(), Deferred()] client.send_produce_request.side_effect = ret client.topic_partitions = {self.topic: [0, 1, 2, 3]} client.metadata_error_for_topic.return_value = False msgs = [self.msg("one"), self.msg("two")] batch_t = 5 clock = MemoryReactorClock() producer = Producer(client, batch_every_t=batch_t, batch_send=True, clock=clock, max_req_attempts=3) # Advance the clock to ensure when no messages to send no error clock.advance(batch_t) d = producer.send_messages(self.topic, msgs=msgs) # Check no request was yet sent self.assertFalse(client.send_produce_request.called) # Advance the clock clock.advance(batch_t) # Check the expected request was sent msgSet = create_message_set( make_send_requests(msgs), producer.codec) req = ProduceRequest(self.topic, 0, msgSet) produce_request_call = call([req], acks=producer.req_acks, timeout=producer.ack_timeout, fail_on_error=False) produce_request_calls = [produce_request_call] client.send_produce_request.assert_has_calls(produce_request_calls) self.assertNoResult(d) # Fire the failure from the first request to the client ret[0].errback(OffsetOutOfRangeError( 'test_producer_send_messages_batched_fail')) # Still no result, producer should retry first self.assertNoResult(d) # Check retry wasn't immediate self.assertEqual(client.send_produce_request.call_count, 1) # Advance the clock by the retry delay clock.advance(producer._retry_interval) # Check 2nd send_produce_request (1st retry) was sent produce_request_calls.append(produce_request_call) client.send_produce_request.assert_has_calls(produce_request_calls) # Fire the failure from the 2nd request to the client ret[1].errback(BrokerNotAvailableError( 'test_producer_send_messages_batched_fail_2')) # Still no result, producer should retry one more time self.assertNoResult(d) # Advance the clock by the retry delay clock.advance(producer._retry_interval * 1.1) # Check 3nd send_produce_request (2st retry) was sent produce_request_calls.append(produce_request_call) client.send_produce_request.assert_has_calls(produce_request_calls) # Fire the failure from the 2nd request to the client ret[2].errback(LeaderNotAvailableError( 'test_producer_send_messages_batched_fail_3')) self.failureResultOf(d, LeaderNotAvailableError) producer.stop()
def test_producer_send_messages_no_retry_fail(self): client = Mock() f = Failure(BrokerNotAvailableError()) client.send_produce_request.side_effect = [fail(f)] client.topic_partitions = {self.topic: [0, 1, 2, 3]} client.metadata_error_for_topic.return_value = False msgs = [self.msg("one"), self.msg("two")] producer = Producer(client, max_req_attempts=1) d = producer.send_messages(self.topic, msgs=msgs) # Check the expected request was sent msgSet = create_message_set( make_send_requests(msgs), producer.codec) req = ProduceRequest(self.topic, 0, msgSet) client.send_produce_request.assert_called_once_with( [req], acks=producer.req_acks, timeout=producer.ack_timeout, fail_on_error=False) self.failureResultOf(d, BrokerNotAvailableError) producer.stop()
def test_producer_send_messages_no_retry_fail(self): client = Mock() f = Failure(BrokerNotAvailableError()) client.send_produce_request.side_effect = [fail(f)] client.topic_partitions = {self.topic: [0, 1, 2, 3]} client.metadata_error_for_topic.return_value = False msgs = [self.msg("one"), self.msg("two")] producer = Producer(client, max_req_attempts=1) d = producer.send_messages(self.topic, msgs=msgs) # Check the expected request was sent msgSet = create_message_set(make_send_requests(msgs), producer.codec) req = ProduceRequest(self.topic, 0, msgSet) client.send_produce_request.assert_called_once_with( [req], acks=producer.req_acks, timeout=producer.ack_timeout, fail_on_error=False) self.failureResultOf(d, BrokerNotAvailableError) producer.stop()
def test_producer_send_messages_batched_fail(self): client = Mock() ret = [Deferred(), Deferred(), Deferred()] client.send_produce_request.side_effect = ret client.topic_partitions = {self.topic: [0, 1, 2, 3]} client.metadata_error_for_topic.return_value = False msgs = [self.msg("one"), self.msg("two")] batch_t = 5 clock = MemoryReactorClock() producer = Producer(client, batch_every_t=batch_t, batch_send=True, clock=clock, max_req_attempts=3) # Advance the clock to ensure when no messages to send no error clock.advance(batch_t) d = producer.send_messages(self.topic, msgs=msgs) # Check no request was yet sent self.assertFalse(client.send_produce_request.called) # Advance the clock clock.advance(batch_t) # Check the expected request was sent msgSet = create_message_set(make_send_requests(msgs), producer.codec) req = ProduceRequest(self.topic, 0, msgSet) produce_request_call = call([req], acks=producer.req_acks, timeout=producer.ack_timeout, fail_on_error=False) produce_request_calls = [produce_request_call] client.send_produce_request.assert_has_calls(produce_request_calls) self.assertNoResult(d) # Fire the failure from the first request to the client ret[0].errback( OffsetOutOfRangeError('test_producer_send_messages_batched_fail')) # Still no result, producer should retry first self.assertNoResult(d) # Check retry wasn't immediate self.assertEqual(client.send_produce_request.call_count, 1) # Advance the clock by the retry delay clock.advance(producer._retry_interval) # Check 2nd send_produce_request (1st retry) was sent produce_request_calls.append(produce_request_call) client.send_produce_request.assert_has_calls(produce_request_calls) # Fire the failure from the 2nd request to the client ret[1].errback( BrokerNotAvailableError( 'test_producer_send_messages_batched_fail_2')) # Still no result, producer should retry one more time self.assertNoResult(d) # Advance the clock by the retry delay clock.advance(producer._retry_interval * 1.1) # Check 3nd send_produce_request (2st retry) was sent produce_request_calls.append(produce_request_call) client.send_produce_request.assert_has_calls(produce_request_calls) # Fire the failure from the 2nd request to the client ret[2].errback( LeaderNotAvailableError( 'test_producer_send_messages_batched_fail_3')) self.failureResultOf(d, LeaderNotAvailableError) producer.stop()