def test_producer_init_simplest(self):
producer = Producer(Mock())
self.assertEqual(
producer.__repr__(),
"<Producer <class 'afkak.partitioner.RoundRobinPartitioner'>:"
"Unbatched:1:1000>")
producer.stop()
def test_producer_send_messages_batched(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f), succeed([ProduceResponse(self.topic, 0, 0, 10L)])]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client,
batch_every_n=batch_n,
batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, ANY, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
self.successResultOf(d)
producer.stop()
def test_producer_send_messages_batched(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f), succeed([ProduceResponse(self.topic, 0, 0, 10L)])]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client, batch_every_n=batch_n, batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, ANY, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=producer.ack_timeout,
fail_on_error=False)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
self.successResultOf(d)
producer.stop()
def test_producer_send_timer_failed(self):
"""test_producer_send_timer_failed
Test that the looping call is restarted when _send_batch errs
Somewhat artificial test to confirm that when failures occur in
_send_batch (which cause the looping call to terminate) that the
looping call is restarted.
"""
client = Mock()
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
batch_t = 5
clock = MemoryReactorClock()
with patch.object(aProducer, 'log') as klog:
producer = Producer(client,
batch_send=True,
batch_every_t=batch_t,
clock=clock)
msgs = [self.msg("one"), self.msg("two")]
d = producer.send_messages(self.topic, msgs=msgs)
# Check no request was yet sent
self.assertFalse(client.send_produce_request.called)
# Patch Producer's Deferred to throw an exception
with patch.object(aProducer, 'Deferred') as d:
d.side_effect = ValueError(
"test_producer_send_timer_failed induced failure")
# Advance the clock
clock.advance(batch_t)
# Check the expected message was logged by the looping call restart
klog.warning.assert_called_once_with('_send_timer_failed:%r: %s',
ANY, ANY)
# Check that the looping call was restarted
self.assertTrue(producer.sendLooper.running)
producer.stop()
def test_producer_send_messages_unknown_topic(self):
client = Mock()
ds = [Deferred() for _ in range(Producer.DEFAULT_REQ_ATTEMPTS)]
clock = MemoryReactorClock()
client.load_metadata_for_topics.side_effect = ds
client.metadata_error_for_topic.return_value = 3
client.topic_partitions = {}
msgs = [self.msg("one"), self.msg("two")]
ack_timeout = 5
producer = Producer(client, ack_timeout=ack_timeout, clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# d is waiting on result from ds[0] for load_metadata_for_topics
self.assertNoResult(d)
# fire it with client still reporting no metadata for topic
# The producer will retry the lookup DEFAULT_REQ_ATTEMPTS times...
for i in range(Producer.DEFAULT_REQ_ATTEMPTS):
ds[i].callback(None)
# And then wait producer._retry_interval for a call back...
clock.advance(producer._retry_interval + 0.01)
self.failureResultOf(d, UnknownTopicOrPartitionError)
self.assertFalse(client.send_produce_request.called)
producer.stop()
def test_producer_send_messages_unknown_topic(self):
client = Mock()
ds = [Deferred() for _ in range(Producer.DEFAULT_REQ_ATTEMPTS)]
clock = MemoryReactorClock()
client.load_metadata_for_topics.side_effect = ds
client.metadata_error_for_topic.return_value = 3
client.topic_partitions = {}
msgs = [self.msg("one"), self.msg("two")]
ack_timeout = 5
producer = Producer(client, ack_timeout=ack_timeout, clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# d is waiting on result from ds[0] for load_metadata_for_topics
self.assertNoResult(d)
# fire it with client still reporting no metadata for topic
# The producer will retry the lookup DEFAULT_REQ_ATTEMPTS times...
for i in range(Producer.DEFAULT_REQ_ATTEMPTS):
ds[i].callback(None)
# And then wait producer._retry_interval for a call back...
clock.advance(producer._retry_interval + 0.01)
self.failureResultOf(d, UnknownTopicOrPartitionError)
self.assertFalse(client.send_produce_request.called)
producer.stop()
def test_producer_send_messages_no_acks(self):
first_part = 19
client = Mock()
ret = Deferred()
client.send_produce_request.return_value = ret
client.topic_partitions = {self.topic: [first_part, 101, 102, 103]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
ack_timeout = 5
producer = Producer(client, ack_timeout=ack_timeout,
req_acks=PRODUCER_ACK_NOT_REQUIRED)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, first_part, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=ack_timeout,
fail_on_error=False)
# Check results when "response" fires
self.assertNoResult(d)
ret.callback([])
result = self.successResultOf(d)
self.assertEqual(result, None)
producer.stop()
def test_producer_init_simplest(self):
producer = Producer(Mock())
self.assertEqual(
producer.__repr__(),
"<Producer <class 'afkak.partitioner.RoundRobinPartitioner'>:"
"Unbatched:1:1000>")
producer.stop()
def test_producer_send_timer_failed(self):
"""test_producer_send_timer_failed
Test that the looping call is restarted when _send_batch errs
Somewhat artificial test to confirm that when failures occur in
_send_batch (which cause the looping call to terminate) that the
looping call is restarted.
"""
client = Mock()
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
batch_t = 5
clock = MemoryReactorClock()
with patch.object(aProducer, 'log') as klog:
producer = Producer(client, batch_send=True, batch_every_t=batch_t,
clock=clock)
msgs = [self.msg("one"), self.msg("two")]
d = producer.send_messages(self.topic, msgs=msgs)
# Check no request was yet sent
self.assertFalse(client.send_produce_request.called)
# Patch Producer's Deferred to throw an exception
with patch.object(aProducer, 'Deferred') as d:
d.side_effect = ValueError(
"test_producer_send_timer_failed induced failure")
# Advance the clock
clock.advance(batch_t)
# Check the expected message was logged by the looping call restart
klog.warning.assert_called_once_with('_send_timer_failed:%r: %s',
ANY, ANY)
# Check that the looping call was restarted
self.assertTrue(producer.sendLooper.running)
producer.stop()
def test_producer_send_messages(self):
first_part = 23
client = Mock()
ret = Deferred()
client.send_produce_request.return_value = ret
client.topic_partitions = {self.topic: [first_part, 101, 102, 103]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
ack_timeout = 5
producer = Producer(client, ack_timeout=ack_timeout)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, first_part, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=ack_timeout,
fail_on_error=False)
# Check results when "response" fires
self.assertNoResult(d)
resp = [ProduceResponse(self.topic, first_part, 0, 10L)]
ret.callback(resp)
result = self.successResultOf(d)
self.assertEqual(result, resp[0])
producer.stop()
def test_producer_cancel_one_request_getting_topic(self):
# Test cancelling a request after it's begun to be processed
client = Mock()
client.topic_partitions = {}
ret = Deferred()
client.load_metadata_for_topics.return_value = ret
msgs = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
batch_n = 4
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
# This will trigger the metadata lookup
d2 = producer.send_messages(self.topic, msgs=msgs2)
d1.cancel()
self.failureResultOf(d1, CancelledError)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
self.assertNoResult(d2)
# Setup the client's topics and trigger the metadata deferred
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
ret.callback(None)
# Expect that only the msgs2 messages were sent
msgSet = create_message_set(
make_send_requests(msgs2), producer.codec)
req = ProduceRequest(self.topic, 1, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=producer.ack_timeout,
fail_on_error=False)
producer.stop()
def test_producer_send_messages_batched_fail(self):
client = Mock()
ret = [Deferred(), Deferred(), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
batch_t = 5
clock = MemoryReactorClock()
producer = Producer(client, batch_every_t=batch_t, batch_send=True,
clock=clock, max_req_attempts=3)
# Advance the clock to ensure when no messages to send no error
clock.advance(batch_t)
d = producer.send_messages(self.topic, msgs=msgs)
# Check no request was yet sent
self.assertFalse(client.send_produce_request.called)
# Advance the clock
clock.advance(batch_t)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
produce_request_call = call([req], acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
produce_request_calls = [produce_request_call]
client.send_produce_request.assert_has_calls(produce_request_calls)
self.assertNoResult(d)
# Fire the failure from the first request to the client
ret[0].errback(OffsetOutOfRangeError(
'test_producer_send_messages_batched_fail'))
# Still no result, producer should retry first
self.assertNoResult(d)
# Check retry wasn't immediate
self.assertEqual(client.send_produce_request.call_count, 1)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval)
# Check 2nd send_produce_request (1st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[1].errback(BrokerNotAvailableError(
'test_producer_send_messages_batched_fail_2'))
# Still no result, producer should retry one more time
self.assertNoResult(d)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval * 1.1)
# Check 3nd send_produce_request (2st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[2].errback(LeaderNotAvailableError(
'test_producer_send_messages_batched_fail_3'))
self.failureResultOf(d, LeaderNotAvailableError)
producer.stop()
def test_producer_send_timer_stopped_error(self):
# Purely for coverage
client = Mock()
producer = Producer(client, batch_send=True)
with patch.object(aProducer, 'log') as klog:
producer._send_timer_stopped('Borg')
klog.warning.assert_called_once_with(
'commitTimerStopped with wrong timer:%s not:%s', 'Borg',
producer.sendLooper)
producer.stop()
def test_producer_init_batch(self):
producer = Producer(Mock(), batch_send=True)
looper = producer.sendLooper
self.assertEqual(type(looper), LoopingCall)
self.assertTrue(looper.running)
producer.stop()
self.assertFalse(looper.running)
self.assertEqual(
producer.__repr__(),
"<Producer <class 'afkak.partitioner.RoundRobinPartitioner'>:"
"10cnt/32768bytes/30secs:1:1000>")
def test_producer_init_batch(self):
producer = Producer(Mock(), batch_send=True)
looper = producer.sendLooper
self.assertEqual(type(looper), LoopingCall)
self.assertTrue(looper.running)
producer.stop()
self.assertFalse(looper.running)
self.assertEqual(
producer.__repr__(),
"<Producer <class 'afkak.partitioner.RoundRobinPartitioner'>:"
"10cnt/32768bytes/30secs:1:1000>")
def test_producer_send_messages_keyed(self):
"""test_producer_send_messages_keyed
Test that messages sent with a key are actually sent with that key
"""
first_part = 43
second_part = 56
client = Mock()
ret1 = Deferred()
client.send_produce_request.side_effect = [ret1]
client.topic_partitions = {self.topic: [first_part, second_part, 102]}
client.metadata_error_for_topic.return_value = False
msgs1 = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
key1 = '35'
key2 = 'foo'
ack_timeout = 5
# Even though we're sending keyed messages, we use the default
# round-robin partitioner, since the requests are easier to predict
producer = Producer(client,
ack_timeout=ack_timeout,
batch_send=True,
batch_every_n=4)
d1 = producer.send_messages(self.topic, key=key1, msgs=msgs1)
d2 = producer.send_messages(self.topic, key=key2, msgs=msgs2)
# Check the expected request was sent
msgSet1 = create_message_set(make_send_requests(msgs1, key=key1),
producer.codec)
msgSet2 = create_message_set(make_send_requests(msgs2, key=key2),
producer.codec)
req1 = ProduceRequest(self.topic, first_part, msgSet1)
req2 = ProduceRequest(self.topic, second_part, msgSet2)
# Annoying, but order of requests is indeterminate...
client.send_produce_request.assert_called_once_with(
ANY,
acks=producer.req_acks,
timeout=ack_timeout,
fail_on_error=False)
self.assertEqual(sorted([req1, req2]),
sorted(client.send_produce_request.call_args[0][0]))
# Check results when "response" fires
self.assertNoResult(d1)
self.assertNoResult(d2)
resp = [
ProduceResponse(self.topic, first_part, 0, 10L),
ProduceResponse(self.topic, second_part, 0, 23L)
]
ret1.callback(resp)
result = self.successResultOf(d1)
self.assertEqual(result, resp[0])
result = self.successResultOf(d2)
self.assertEqual(result, resp[1])
producer.stop()
def test_producer_send_messages_unexpected_err(self):
client = Mock()
f = Failure(TypeError())
client.send_produce_request.side_effect = [fail(f)]
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client)
with patch.object(aProducer, 'log') as klog:
d = producer.send_messages(self.topic, msgs=msgs)
klog.error.assert_called_once_with(
'Unexpected failure: %r in _handle_send_response', f)
self.failureResultOf(d, TypeError)
producer.stop()
def test_producer_send_messages_unexpected_err(self):
client = Mock()
f = Failure(TypeError())
client.send_produce_request.side_effect = [fail(f)]
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client)
with patch.object(aProducer, 'log') as klog:
d = producer.send_messages(self.topic, msgs=msgs)
klog.error.assert_called_once_with(
'Unexpected failure: %r in _handle_send_response', f)
self.failureResultOf(d, TypeError)
producer.stop()
def test_producer_send_messages_keyed(self):
"""test_producer_send_messages_keyed
Test that messages sent with a key are actually sent with that key
"""
first_part = 43
second_part = 56
client = Mock()
ret1 = Deferred()
client.send_produce_request.side_effect = [ret1]
client.topic_partitions = {self.topic: [first_part, second_part, 102]}
client.metadata_error_for_topic.return_value = False
msgs1 = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
key1 = '35'
key2 = 'foo'
ack_timeout = 5
# Even though we're sending keyed messages, we use the default
# round-robin partitioner, since the requests are easier to predict
producer = Producer(client, ack_timeout=ack_timeout, batch_send=True,
batch_every_n=4)
d1 = producer.send_messages(self.topic, key=key1, msgs=msgs1)
d2 = producer.send_messages(self.topic, key=key2, msgs=msgs2)
# Check the expected request was sent
msgSet1 = create_message_set(
make_send_requests(msgs1, key=key1), producer.codec)
msgSet2 = create_message_set(
make_send_requests(msgs2, key=key2), producer.codec)
req1 = ProduceRequest(self.topic, first_part, msgSet1)
req2 = ProduceRequest(self.topic, second_part, msgSet2)
# Annoying, but order of requests is indeterminate...
client.send_produce_request.assert_called_once_with(
ANY, acks=producer.req_acks, timeout=ack_timeout,
fail_on_error=False)
self.assertEqual(sorted([req1, req2]),
sorted(client.send_produce_request.call_args[0][0]))
# Check results when "response" fires
self.assertNoResult(d1)
self.assertNoResult(d2)
resp = [ProduceResponse(self.topic, first_part, 0, 10L),
ProduceResponse(self.topic, second_part, 0, 23L)]
ret1.callback(resp)
result = self.successResultOf(d1)
self.assertEqual(result, resp[0])
result = self.successResultOf(d2)
self.assertEqual(result, resp[1])
producer.stop()
def __init__(self, config):
self._topics = config['kafka']['topics']
kafka_client = KafkaClient(
Config.get('kafka')['server'] + ':' +
str(Config.get('kafka')['port']))
self._producer = Producer(kafka_client)
def test_producer_complete_batch_send_unexpected_error(self):
# Purely for coverage
client = Mock()
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
e = ValueError('test_producer_complete_batch_send_unexpected_error')
client.send_produce_request.side_effect = e
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client)
with patch.object(aProducer, 'log') as klog:
producer.send_messages(self.topic, msgs=msgs)
# The error 'e' gets wrapped in a failure with a traceback, so
# we can't easily match the call exactly...
klog.error.assert_called_once_with(
'Failure detected in _complete_batch_send: %r\n%r', ANY, ANY)
producer.stop()
def test_producer_send_messages_no_retry_fail(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
client.send_produce_request.side_effect = [fail(f)]
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client, max_req_attempts=1)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(
make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
client.send_produce_request.assert_called_once_with(
[req], acks=producer.req_acks, timeout=producer.ack_timeout,
fail_on_error=False)
self.failureResultOf(d, BrokerNotAvailableError)
producer.stop()
def test_producer_send_messages_no_retry_fail(self):
client = Mock()
f = Failure(BrokerNotAvailableError())
client.send_produce_request.side_effect = [fail(f)]
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client, max_req_attempts=1)
d = producer.send_messages(self.topic, msgs=msgs)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
self.failureResultOf(d, BrokerNotAvailableError)
producer.stop()
def test_producer_stop_during_request(self):
# Test stopping producer while it's waiting for reply from client
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client, batch_every_n=batch_n, batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
producer.stop()
self.failureResultOf(d, tid_CancelledError)
def test_producer_cancel_request_in_batch(self):
# Test cancelling a request before it's begun to be processed
client = Mock()
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
batch_n = 3
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
d1.cancel()
self.failureResultOf(d1, CancelledError)
d2 = producer.send_messages(self.topic, msgs=msgs2)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
self.assertNoResult(d2)
producer.stop()
def test_producer_cancel_one_request_getting_topic(self):
# Test cancelling a request after it's begun to be processed
client = Mock()
client.topic_partitions = {}
ret = Deferred()
client.load_metadata_for_topics.return_value = ret
msgs = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
batch_n = 4
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
# This will trigger the metadata lookup
d2 = producer.send_messages(self.topic, msgs=msgs2)
d1.cancel()
self.failureResultOf(d1, CancelledError)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
self.assertNoResult(d2)
# Setup the client's topics and trigger the metadata deferred
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
ret.callback(None)
# Expect that only the msgs2 messages were sent
msgSet = create_message_set(make_send_requests(msgs2), producer.codec)
req = ProduceRequest(self.topic, 1, msgSet)
client.send_produce_request.assert_called_once_with(
[req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
producer.stop()
def test_producer_stop_during_request(self):
"""
Test stopping producer while it's waiting for reply from client
"""
clock = MemoryReactorClock()
client = Mock(reactor=clock)
f = Failure(BrokerNotAvailableError())
ret = [fail(f), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
batch_n = 2
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d = producer.send_messages(self.topic, msgs=msgs)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
clock.advance(producer._retry_interval)
producer.stop()
self.failureResultOf(d, tid_CancelledError)
def produce(reactor, hosts="localhost:9092"):
topic = b"example_topic"
stop_time = reactor.seconds() + 60.0 # seconds to run
client = yield ready_client(reactor, hosts, topic)
producers = [
Producer(client, RoundRobinPartitioner),
Producer(client, HashedPartitioner),
]
def cb_produce(resp):
log.info("Produce got response: %r", resp)
def eb_produce(f):
log.error("Produce failed",
exc_info=(f.type, f.value, f.getTracebackObject()))
while reactor.seconds() < stop_time:
ds = [
# Wait at least half a second before the next send.
task.deferLater(reactor, 0.5, lambda: None),
]
# Create some number of random messages and send them to
# the producers in parallel.
for producer in producers:
key, messages = make_messages()
d = producer.send_messages(topic, key=key, msgs=messages)
d.addCallbacks(cb_produce, eb_produce)
ds.append(d)
yield defer.gatherResults(ds)
log.info("\n")
log.info("Time is up, stopping producers...")
for p in producers:
p.stop()
yield client.close()
def test_producer_cancel_getting_topic(self):
# Test cancelling while waiting to retry getting metadata
clock = MemoryReactorClock()
client = Mock()
client.topic_partitions = {} # start with no metadata
rets = [Deferred(), Deferred()]
client.load_metadata_for_topics.side_effect = rets
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client, clock=clock)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
# Fire the result of load_metadata_for_topics, but
# metadata_error_for_topic is still True, so it'll retry after delay
# Advance the clock, some, but not enough to retry
rets[0].callback(None)
# Advance to partway thru the delay
clock.advance(producer._retry_interval / 2)
# Cancel the request and ake sure we got the CancelledError
d1.cancel()
self.failureResultOf(d1, CancelledError)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
# Setup the client's topics and trigger the metadata deferred
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
rets[1].callback(None)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
# Advance the clock again to complete the delay
clock.advance(producer._retry_interval)
# Make sure the retry got reset
self.assertEqual(producer._retry_interval,
producer._init_retry_interval)
producer.stop()
def test_producer_cancel_getting_topic(self):
# Test cancelling while waiting to retry getting metadata
clock = MemoryReactorClock()
client = Mock()
client.topic_partitions = {} # start with no metadata
rets = [Deferred(), Deferred()]
client.load_metadata_for_topics.side_effect = rets
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client, clock=clock)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
# Fire the result of load_metadata_for_topics, but
# metadata_error_for_topic is still True, so it'll retry after delay
# Advance the clock, some, but not enough to retry
rets[0].callback(None)
# Advance to partway thru the delay
clock.advance(producer._retry_interval / 2)
# Cancel the request and ake sure we got the CancelledError
d1.cancel()
self.failureResultOf(d1, CancelledError)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
# Setup the client's topics and trigger the metadata deferred
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
rets[1].callback(None)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
# Advance the clock again to complete the delay
clock.advance(producer._retry_interval)
# Make sure the retry got reset
self.assertEqual(producer._retry_interval,
producer._init_retry_interval)
producer.stop()
def test_producer_stop_waiting_to_retry(self):
# Test stopping producer while it's waiting to retry a request
client = Mock()
f = Failure(BrokerNotAvailableError())
ret = [fail(f)]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
clock = MemoryReactorClock()
batch_n = 2
producer = Producer(client,
batch_every_n=batch_n,
batch_send=True,
clock=clock)
d = producer.send_messages(self.topic, msgs=msgs)
# At first, there's no result. Have to retry due to first failure
self.assertNoResult(d)
# Advance the clock, some, but not enough to retry
clock.advance(producer._retry_interval / 2)
# Stop the producer before the retry
producer.stop()
self.failureResultOf(d, tid_CancelledError)
def __init__(self, config):
self._topics = config['kafka']['topics']
kafka_client = KafkaClient(
Config.get('kafka')['server'] + ':' +
str(Config.get('kafka')['port']))
self._producer = Producer(kafka_client)
self._dbpools = {}
for database_name in config['databases']:
db_config = Config.get('databases')[database_name]
self._dbpools[database_name] = ReconnectingConnectionPool(
"pymysql",
host=db_config['host'],
user=db_config['user'],
password=db_config['password'],
db=db_config['db'],
charset=db_config['charset'],
autocommit=True,
cursorclass=pymysql.cursors.DictCursor)
def test_producer_send_timer_stopped_error(self):
# Purely for coverage
client = Mock()
producer = Producer(client, batch_send=True)
with patch.object(aProducer, 'log') as klog:
producer._send_timer_stopped('Borg')
klog.warning.assert_called_once_with(
'commitTimerStopped with wrong timer:%s not:%s', 'Borg',
producer.sendLooper)
producer.stop()
def test_producer_complete_batch_send_unexpected_error(self):
# Purely for coverage
client = Mock()
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
e = ValueError('test_producer_complete_batch_send_unexpected_error')
client.send_produce_request.side_effect = e
msgs = [self.msg("one"), self.msg("two")]
producer = Producer(client)
with patch.object(aProducer, 'log') as klog:
producer.send_messages(self.topic, msgs=msgs)
# The error 'e' gets wrapped in a failure with a traceback, so
# we can't easily match the call exactly...
klog.error.assert_called_once_with(
'Failure detected in _complete_batch_send: %r\n%r', ANY, ANY)
producer.stop()
def test_producer_cancel_request_in_batch(self):
# Test cancelling a request before it's begun to be processed
client = Mock()
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
msgs2 = [self.msg("three"), self.msg("four")]
batch_n = 3
producer = Producer(client, batch_every_n=batch_n, batch_send=True)
d1 = producer.send_messages(self.topic, msgs=msgs)
# Check that no request was sent
self.assertFalse(client.send_produce_request.called)
d1.cancel()
self.failureResultOf(d1, CancelledError)
d2 = producer.send_messages(self.topic, msgs=msgs2)
# Check that still no request was sent
self.assertFalse(client.send_produce_request.called)
self.assertNoResult(d2)
producer.stop()
def test_producer_bad_codec_value(self):
with self.assertRaises(UnsupportedCodecError):
Producer(Mock(), codec=99)
class ProducerSendMessagesValidationTests(unittest.SynchronousTestCase):
"""
Test the validation `afkak.producer.Producer.send_messages()` applies to
its arguments.
:ivar producer: `Producer` with default arguments.
"""
def setUp(self):
client = Mock(reactor=MemoryReactorClock())
self.producer = Producer(client)
self.addCleanup(self.producer.stop)
def test_topic_type(self):
"""
`TypeError` results when the *topic* argument is not text.
"""
self.failureResultOf(self.producer.send_messages(1234, msgs=[b'']),
TypeError)
def test_topic_bytes(self):
"""
`TypeError` results when the *topic* argument is a bytestring on Python 3.
"""
if type('') is type(b''):
raise unittest.SkipTest('str is bytes on Python 2')
self.failureResultOf(self.producer.send_messages(b'topic', msgs=[b'']),
TypeError)
def test_empty_messages(self):
"""
`ValueError` results when the *msgs* argument is not passed or is
empty.
"""
self.failureResultOf(self.producer.send_messages('topic'), ValueError)
self.failureResultOf(self.producer.send_messages('topic', msgs=[]),
ValueError)
def test_message_type(self):
"""
`TypeError` results when members of the *msgs* sequence are not
`bytes` or ``None``.
"""
self.failureResultOf(
self.producer.send_messages('topic', msgs=[1, 2, 3]), TypeError)
self.failureResultOf(
self.producer.send_messages('topic', msgs=[u'asdf']), TypeError)
def test_none_message(self):
"""
A message may be ``None``. This doesn't make much sense unless there is
also a key.
"""
d = self.producer.send_messages('topic', key=b'key', msgs=[None])
d.addErrback(lambda f: None
) # Handle the cancellation failure from producer.stop().
self.assertNoResult(d)
def test_key_type(self):
"""
The key must not be unicode, but bytes.
"""
self.failureResultOf(
self.producer.send_messages('topic', key=u'key', msgs=[b'msg']),
TypeError)
def test_producer_bad_codec_type(self):
with self.assertRaises(TypeError):
Producer(Mock(), codec='bogus')
def test_producer_send_empty_messages(self):
client = Mock()
producer = Producer(client)
d = producer.send_messages(self.topic)
self.failureResultOf(d, ValueError)
producer.stop()
def setUp(self):
client = Mock(reactor=MemoryReactorClock())
self.producer = Producer(client)
self.addCleanup(self.producer.stop)
def test_producer_send_messages_batched_partial_success(self):
"""test_producer_send_messages_batched_partial_success
This tests the complexity of the error handling for a single batch
request.
Scenario: The producer's caller sends 5 requests to two (total) topics
The client's metadata is such that the producer will produce
requests to post msgs to 5 separate topic/partition tuples
The batch size is reached, so the producer sends the request
The caller then cancels one of the requests
The (mock) client returns partial success in the form of a
FailedPayloadsError.
The Producer then should return the successful results and
retry the failed.
The (mock) client then "succeeds" the remaining results.
"""
client = Mock()
topic2 = 'tpsmbps_two'
client.topic_partitions = {self.topic: [0, 1, 2, 3], topic2: [4, 5, 6]}
client.metadata_error_for_topic.return_value = False
init_resp = [
ProduceResponse(self.topic, 0, 0, 10L),
ProduceResponse(self.topic, 1, 6, 20L),
ProduceResponse(topic2, 5, 0, 30L),
]
next_resp = [
ProduceResponse(self.topic, 2, 0, 10L),
ProduceResponse(self.topic, 1, 0, 20L),
ProduceResponse(topic2, 4, 0, 30L),
]
failed_payloads = [
(ProduceRequest(self.topic, ANY,
ANY), NotLeaderForPartitionError()),
(ProduceRequest(topic2, ANY, ANY), BrokerNotAvailableError()),
]
f = Failure(FailedPayloadsError(init_resp, failed_payloads))
ret = [fail(f), succeed(next_resp)]
client.send_produce_request.side_effect = ret
msgs = self.msgs(range(10))
results = []
clock = MemoryReactorClock()
producer = Producer(client,
batch_send=True,
batch_every_t=0,
clock=clock)
# Send 5 total requests: 4 here, one after we make sure we didn't
# send early
results.append(producer.send_messages(self.topic, msgs=msgs[0:3]))
results.append(producer.send_messages(topic2, msgs=msgs[3:5]))
results.append(producer.send_messages(self.topic, msgs=msgs[5:8]))
results.append(producer.send_messages(topic2, msgs=msgs[8:9]))
# No call yet, not enough messages
self.assertFalse(client.send_produce_request.called)
# Enough messages to start the request
results.append(producer.send_messages(self.topic, msgs=msgs[9:10]))
# Before the retry, there should be some results
self.assertEqual(init_resp[0], self.successResultOf(results[0]))
self.assertEqual(init_resp[2], self.successResultOf(results[3]))
# Advance the clock
clock.advance(producer._retry_interval)
# Check the otehr results came in
self.assertEqual(next_resp[0], self.successResultOf(results[4]))
self.assertEqual(next_resp[1], self.successResultOf(results[2]))
self.assertEqual(next_resp[2], self.successResultOf(results[1]))
producer.stop()
def test_producer_bad_codec_value(self):
with self.assertRaises(UnsupportedCodecError):
p = Producer(Mock(), codec=99)
p.__repr__() # pragma: no cover # STFU pyflakes
def test_producer_bad_codec_type(self):
with self.assertRaises(TypeError):
p = Producer(Mock(), codec='bogus')
p.__repr__() # pragma: no cover # STFU pyflakes
def test_producer_non_integral_batch_every_b(self):
client = Mock()
with self.assertRaises(TypeError):
producer = Producer(client, batch_send=True, batch_every_b="10")
producer.__repr__() # pragma: no cover # STFU pyflakes
def test_producer_send_messages_batched_partial_success(self):
"""test_producer_send_messages_batched_partial_success
This tests the complexity of the error handling for a single batch
request.
Scenario: The producer's caller sends 5 requests to two (total) topics
The client's metadata is such that the producer will produce
requests to post msgs to 5 separate topic/partition tuples
The batch size is reached, so the producer sends the request
The caller then cancels one of the requests
The (mock) client returns partial success in the form of a
FailedPayloadsError.
The Producer then should return the successful results and
retry the failed.
The (mock) client then "succeeds" the remaining results.
"""
client = Mock()
topic2 = 'tpsmbps_two'
client.topic_partitions = {self.topic: [0, 1, 2, 3], topic2: [4, 5, 6]}
client.metadata_error_for_topic.return_value = False
init_resp = [ProduceResponse(self.topic, 0, 0, 10L),
ProduceResponse(self.topic, 1, 6, 20L),
ProduceResponse(topic2, 5, 0, 30L),
]
next_resp = [ProduceResponse(self.topic, 2, 0, 10L),
ProduceResponse(self.topic, 1, 0, 20L),
ProduceResponse(topic2, 4, 0, 30L),
]
failed_payloads = [(ProduceRequest(self.topic, ANY, ANY),
NotLeaderForPartitionError()),
(ProduceRequest(topic2, ANY, ANY),
BrokerNotAvailableError()),
]
f = Failure(FailedPayloadsError(init_resp, failed_payloads))
ret = [fail(f), succeed(next_resp)]
client.send_produce_request.side_effect = ret
msgs = self.msgs(range(10))
results = []
clock = MemoryReactorClock()
producer = Producer(client, batch_send=True, batch_every_t=0,
clock=clock)
# Send 5 total requests: 4 here, one after we make sure we didn't
# send early
results.append(producer.send_messages(self.topic, msgs=msgs[0:3]))
results.append(producer.send_messages(topic2, msgs=msgs[3:5]))
results.append(producer.send_messages(self.topic, msgs=msgs[5:8]))
results.append(producer.send_messages(topic2, msgs=msgs[8:9]))
# No call yet, not enough messages
self.assertFalse(client.send_produce_request.called)
# Enough messages to start the request
results.append(producer.send_messages(self.topic, msgs=msgs[9:10]))
# Before the retry, there should be some results
self.assertEqual(init_resp[0], self.successResultOf(results[0]))
self.assertEqual(init_resp[2], self.successResultOf(results[3]))
# Advance the clock
clock.advance(producer._retry_interval)
# Check the otehr results came in
self.assertEqual(next_resp[0], self.successResultOf(results[4]))
self.assertEqual(next_resp[1], self.successResultOf(results[2]))
self.assertEqual(next_resp[2], self.successResultOf(results[1]))
producer.stop()
def test_producer_send_messages_batched_fail(self):
client = Mock()
ret = [Deferred(), Deferred(), Deferred()]
client.send_produce_request.side_effect = ret
client.topic_partitions = {self.topic: [0, 1, 2, 3]}
client.metadata_error_for_topic.return_value = False
msgs = [self.msg("one"), self.msg("two")]
batch_t = 5
clock = MemoryReactorClock()
producer = Producer(client,
batch_every_t=batch_t,
batch_send=True,
clock=clock,
max_req_attempts=3)
# Advance the clock to ensure when no messages to send no error
clock.advance(batch_t)
d = producer.send_messages(self.topic, msgs=msgs)
# Check no request was yet sent
self.assertFalse(client.send_produce_request.called)
# Advance the clock
clock.advance(batch_t)
# Check the expected request was sent
msgSet = create_message_set(make_send_requests(msgs), producer.codec)
req = ProduceRequest(self.topic, 0, msgSet)
produce_request_call = call([req],
acks=producer.req_acks,
timeout=producer.ack_timeout,
fail_on_error=False)
produce_request_calls = [produce_request_call]
client.send_produce_request.assert_has_calls(produce_request_calls)
self.assertNoResult(d)
# Fire the failure from the first request to the client
ret[0].errback(
OffsetOutOfRangeError('test_producer_send_messages_batched_fail'))
# Still no result, producer should retry first
self.assertNoResult(d)
# Check retry wasn't immediate
self.assertEqual(client.send_produce_request.call_count, 1)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval)
# Check 2nd send_produce_request (1st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[1].errback(
BrokerNotAvailableError(
'test_producer_send_messages_batched_fail_2'))
# Still no result, producer should retry one more time
self.assertNoResult(d)
# Advance the clock by the retry delay
clock.advance(producer._retry_interval * 1.1)
# Check 3nd send_produce_request (2st retry) was sent
produce_request_calls.append(produce_request_call)
client.send_produce_request.assert_has_calls(produce_request_calls)
# Fire the failure from the 2nd request to the client
ret[2].errback(
LeaderNotAvailableError(
'test_producer_send_messages_batched_fail_3'))
self.failureResultOf(d, LeaderNotAvailableError)
producer.stop()
def test_producer_send_empty_messages(self):
client = Mock()
producer = Producer(client)
d = producer.send_messages(self.topic)
self.failureResultOf(d, ValueError)
producer.stop()
def test_producer_bad_codec_type(self):
with self.assertRaises(TypeError):
p = Producer(Mock(), codec='bogus')
p.__repr__() # pragma: no cover # STFU pyflakes
def test_producer_non_integral_batch_every_b(self):
client = Mock()
with self.assertRaises(TypeError):
producer = Producer(client, batch_send=True, batch_every_b="10")
producer.__repr__() # pragma: no cover # STFU pyflakes
def test_producer_bad_codec_value(self):
with self.assertRaises(UnsupportedCodecError):
p = Producer(Mock(), codec=99)
p.__repr__() # pragma: no cover # STFU pyflakes
class ExampleMixedAction(Action):
def __init__(self, config):
self._topics = config['kafka']['topics']
kafka_client = KafkaClient(
Config.get('kafka')['server'] + ':' +
str(Config.get('kafka')['port']))
self._producer = Producer(kafka_client)
self._dbpools = {}
for database_name in config['databases']:
db_config = Config.get('databases')[database_name]
self._dbpools[database_name] = ReconnectingConnectionPool(
"pymysql",
host=db_config['host'],
user=db_config['user'],
password=db_config['password'],
db=db_config['db'],
charset=db_config['charset'],
autocommit=True,
cursorclass=pymysql.cursors.DictCursor)
def execute(self, request):
source = request['client']
message = request['msg']
timestamp = request['timestamp']
if message['type'] == 'event':
return self.__sink_event(source, message['data'], timestamp)
elif message['type'] == 'info':
return self._dbpools['my_db'].runInteraction(
self.__sink_info, source, message['data'], timestamp)
def __sink_event(self, source, data, timestamp):
event = {
'source':
source,
'event':
list(data.keys())[0],
'timestamp':
datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
}
return self._producer.send_messages(self._topics['events'],
msgs=[json.dumps(event).encode()])
def __sink_info(self, adapi_transaction, source, data, timestamp):
info = [
source, data['Temperature'], data['Humidity'], data['Light'],
datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
]
adapi_transaction.execute(
"INSERT INTO measurements (client, temperature, humidity, light, created_at) VALUES (%s, %s, %s, %s, %s)",
info)
adapi_transaction.execute("SELECT LAST_INSERT_ID() as inserted_id")
result = adapi_transaction.fetchone()
return result['inserted_id']
