def test_producer_sync_fail_on_error(self): error = FailedPayloadsError('failure') with patch.object(SimpleClient, 'load_metadata_for_topics'): with patch.object(SimpleClient, 'ensure_topic_exists'): with patch.object(SimpleClient, 'get_partition_ids_for_topic', return_value=[0, 1]): with patch.object(SimpleClient, '_send_broker_aware_request', return_value=[error]): client = SimpleClient(MagicMock()) producer = SimpleProducer(client, async_send=False, sync_fail_on_error=False) # This should not raise (response, ) = producer.send_messages( 'foobar', b'test message') self.assertEqual(response, error) producer = SimpleProducer(client, async_send=False, sync_fail_on_error=True) with self.assertRaises(FailedPayloadsError): producer.send_messages('foobar', b'test message')
def send_side_effect(reqs, *args, **kwargs): if self.client.is_first_time: self.client.is_first_time = False return [FailedPayloadsError(req) for req in reqs] responses = [] for req in reqs: offset = offsets[req.topic][req.partition] offsets[req.topic][req.partition] += len(req.messages) responses.append( ProduceResponsePayload(req.topic, req.partition, 0, offset) ) return responses
def failed_payloads(payload): return FailedPayloadsError(payload)
def mock_offset_request(payloads, **kwargs): raise FailedPayloadsError(payloads[0])
def _send_consumer_aware_request(self, group, payloads, encoder_fn, decoder_fn): """ Send a list of requests to the consumer coordinator for the group specified using the supplied encode/decode functions. As the payloads that use consumer-aware requests do not contain the group (e.g. OffsetFetchRequest), all payloads must be for a single group. Arguments: group: the name of the consumer group (str) the payloads are for payloads: list of object-like entities with topic (str) and partition (int) attributes; payloads with duplicate topic+partition are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] broker = self._get_coordinator_for_group(group) # Send the list of request payloads and collect the responses and # errors responses = {} request_id = self._next_id() log.debug('Request %s to %s: %s', request_id, broker, payloads) request = encoder_fn(client_id=self.client_id, correlation_id=request_id, payloads=payloads) # Send the request, recv the response try: host, port, afi = get_ip_port_afi(broker.host) conn = self._get_conn(host, broker.port, afi) conn.send(request_id, request) except ConnectionError as e: log.warning( 'ConnectionError attempting to send request %s ' 'to server %s: %s', request_id, broker, e) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) # No exception, try to get response else: # decoder_fn=None signal that the server is expected to not # send a response. This probably only applies to # ProduceRequest w/ acks = 0 if decoder_fn is None: log.debug( 'Request %s does not expect a response ' '(skipping conn.recv)', request_id) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = None return [] try: response = conn.recv(request_id) except ConnectionError as e: log.warning( 'ConnectionError attempting to receive a ' 'response to request %s from server %s: %s', request_id, broker, e) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) else: _resps = [] for payload_response in decoder_fn(response): topic_partition = (payload_response.topic, payload_response.partition) responses[topic_partition] = payload_response _resps.append(payload_response) log.debug('Response %s: %s', request_id, _resps) # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def failed_payloads(payloads): for payload in payloads: topic_partition = (str(payload.topic), payload.partition) responses[(topic_partition)] = FailedPayloadsError(payload)
def send_side_effect(reqs, *args, **kwargs): return [FailedPayloadsError(req) for req in reqs]