def test_producer_sync_fail_on_error(self):
error = FailedPayloadsError('failure')
with patch.object(SimpleClient, 'load_metadata_for_topics'):
with patch.object(SimpleClient, 'ensure_topic_exists'):
with patch.object(SimpleClient,
'get_partition_ids_for_topic',
return_value=[0, 1]):
with patch.object(SimpleClient,
'_send_broker_aware_request',
return_value=[error]):
client = SimpleClient(MagicMock())
producer = SimpleProducer(client,
async_send=False,
sync_fail_on_error=False)
# This should not raise
(response, ) = producer.send_messages(
'foobar', b'test message')
self.assertEqual(response, error)
producer = SimpleProducer(client,
async_send=False,
sync_fail_on_error=True)
with self.assertRaises(FailedPayloadsError):
producer.send_messages('foobar', b'test message')
def send_side_effect(reqs, *args, **kwargs):
if self.client.is_first_time:
self.client.is_first_time = False
return [FailedPayloadsError(req) for req in reqs]
responses = []
for req in reqs:
offset = offsets[req.topic][req.partition]
offsets[req.topic][req.partition] += len(req.messages)
responses.append(
ProduceResponsePayload(req.topic, req.partition, 0, offset)
)
return responses
def failed_payloads(payload):
return FailedPayloadsError(payload)
def mock_offset_request(payloads, **kwargs):
raise FailedPayloadsError(payloads[0])
def _send_consumer_aware_request(self, group, payloads, encoder_fn,
decoder_fn):
"""
Send a list of requests to the consumer coordinator for the group
specified using the supplied encode/decode functions. As the payloads
that use consumer-aware requests do not contain the group (e.g.
OffsetFetchRequest), all payloads must be for a single group.
Arguments:
group: the name of the consumer group (str) the payloads are for
payloads: list of object-like entities with topic (str) and
partition (int) attributes; payloads with duplicate
topic+partition are not supported.
encode_fn: a method to encode the list of payloads to a request body,
must accept client_id, correlation_id, and payloads as
keyword arguments
decode_fn: a method to decode a response body into response objects.
The response objects must be object-like and have topic
and partition attributes
Returns:
List of response objects in the same order as the supplied payloads
"""
# encoders / decoders do not maintain ordering currently
# so we need to keep this so we can rebuild order before returning
original_ordering = [(p.topic, p.partition) for p in payloads]
broker = self._get_coordinator_for_group(group)
# Send the list of request payloads and collect the responses and
# errors
responses = {}
request_id = self._next_id()
log.debug('Request %s to %s: %s', request_id, broker, payloads)
request = encoder_fn(client_id=self.client_id,
correlation_id=request_id,
payloads=payloads)
# Send the request, recv the response
try:
host, port, afi = get_ip_port_afi(broker.host)
conn = self._get_conn(host, broker.port, afi)
conn.send(request_id, request)
except ConnectionError as e:
log.warning(
'ConnectionError attempting to send request %s '
'to server %s: %s', request_id, broker, e)
for payload in payloads:
topic_partition = (payload.topic, payload.partition)
responses[topic_partition] = FailedPayloadsError(payload)
# No exception, try to get response
else:
# decoder_fn=None signal that the server is expected to not
# send a response. This probably only applies to
# ProduceRequest w/ acks = 0
if decoder_fn is None:
log.debug(
'Request %s does not expect a response '
'(skipping conn.recv)', request_id)
for payload in payloads:
topic_partition = (payload.topic, payload.partition)
responses[topic_partition] = None
return []
try:
response = conn.recv(request_id)
except ConnectionError as e:
log.warning(
'ConnectionError attempting to receive a '
'response to request %s from server %s: %s', request_id,
broker, e)
for payload in payloads:
topic_partition = (payload.topic, payload.partition)
responses[topic_partition] = FailedPayloadsError(payload)
else:
_resps = []
for payload_response in decoder_fn(response):
topic_partition = (payload_response.topic,
payload_response.partition)
responses[topic_partition] = payload_response
_resps.append(payload_response)
log.debug('Response %s: %s', request_id, _resps)
# Return responses in the same order as provided
return [responses[tp] for tp in original_ordering]
def failed_payloads(payloads):
for payload in payloads:
topic_partition = (str(payload.topic), payload.partition)
responses[(topic_partition)] = FailedPayloadsError(payload)
def send_side_effect(reqs, *args, **kwargs):
return [FailedPayloadsError(req) for req in reqs]