def test_producer_sync_fail_on_error(self): error = FailedPayloadsError('failure') with patch.object(KafkaClient, 'load_metadata_for_topics'): with patch.object(KafkaClient, 'get_partition_ids_for_topic', return_value=[0, 1]): with patch.object(KafkaClient, '_send_broker_aware_request', return_value=[error]): client = KafkaClient(MagicMock()) producer = SimpleProducer(client, async=False, sync_fail_on_error=False) # This should not raise (response, ) = producer.send_messages( 'foobar', b'test message') self.assertEqual(response, error) producer = SimpleProducer(client, async=False, sync_fail_on_error=True) with self.assertRaises(FailedPayloadsError): producer.send_messages('foobar', b'test message')
def test_populate_topic_to_offset_map(self, producer, topic): response_one = ProduceResponse(topic, partition=0, error=0, offset=1) response_two = FailedPayloadsError(payload=mock.Mock()) responses = [response_one, response_two] topics_map = producer._kafka_producer._populate_topics_to_offset_map( responses) assert len(topics_map) == 1 assert topic in topics_map
def send_side_effect(reqs, *args, **kwargs): if self.client.is_first_time: self.client.is_first_time = False return [FailedPayloadsError(req) for req in reqs] responses = [] for req in reqs: offset = offsets[req.topic][req.partition] offsets[req.topic][req.partition] += len(req.messages) responses.append( ProduceResponse(req.topic, req.partition, 0, offset)) return responses
def test_offset_retry_on_network_flake(self, consumer_instance): mock_offsets = {'test-topic': {0: 10}} exception = FailedPayloadsError("Network flake!") with mock.patch.object( consumer_instance.kafka_client, 'send_offset_commit_request', side_effect=[exception, exception, exception, None]) as mock_send_offsets, mock.patch.object( consumer_instance, '_get_offsets_map_to_be_committed', return_value=mock_offsets): consumer_instance.commit_offsets(mock_offsets) assert mock_send_offsets.call_count == 4
def test_publish_one_msg_succeeds_one_fails_after_retry( self, message, another_message, topic, producer): # TODO(DATAPIPE-606|clin) investigate better way than mocking response mock_response = ProduceResponse(topic, partition=0, error=0, offset=1) fail_response = FailedPayloadsError(payload=mock.Mock()) side_effect = ([[mock_response, fail_response]] + [[fail_response]] * self.max_retry_count) with mock.patch.object( producer._kafka_producer.kafka_client, 'send_produce_request', side_effect=side_effect), pytest.raises(MaxRetryError) as e: producer.publish(message) producer.publish(another_message) producer.flush() self.assert_last_retry_result(e.value.last_result, another_message, expected_published_msgs_count=1)
failed = True except ConnectionError, e: log.warning("Could not send request [%s] to server %s: %s", request, conn, e) failed = True if failed: failed_payloads += payloads self.reset_all_metadata() continue for response in decoder_fn(response): acc[(response.topic, response.partition)] = response if failed_payloads: raise FailedPayloadsError(failed_payloads) # Order the accumulated responses by the original key order return (acc[k] for k in original_keys) if acc else () def __repr__(self): return '<KafkaClient client_id=%s>' % (self.client_id) def _raise_on_response_error(self, resp): if resp.error == ErrorMapping.NO_ERROR: return if resp.error in (ErrorMapping.UNKNOWN_TOPIC_OR_PARTITON, ErrorMapping.NOT_LEADER_FOR_PARTITION): self.reset_topic_metadata(resp.topic)
def mock_offset_commit_request(group, payloads, **kwargs): raise FailedPayloadsError(payloads[0])
def failed_payloads(payload): return FailedPayloadsError(payload)
def _send_broker_aware_request(self, payloads, encoder_fn, decoder_fn): """ Group a list of request payloads by topic+partition and send them to the leader broker for that partition using the supplied encode/decode functions Params ====== payloads: list of object-like entities with a topic (str) and partition (int) attribute encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Return ====== List of response objects in the same order as the supplied payloads """ # Group the requests by topic+partition original_keys = [] payloads_by_broker = collections.defaultdict(list) for payload in payloads: leader = self._get_leader_for_partition(payload.topic, payload.partition) payloads_by_broker[leader].append(payload) original_keys.append((payload.topic, payload.partition)) # Accumulate the responses in a dictionary acc = {} # keep a list of payloads that were failed to be sent to brokers failed_payloads = [] # For each broker, send the list of request payloads for broker, payloads in payloads_by_broker.items(): conn = self._get_conn(broker.host.decode('utf-8'), broker.port) requestId = self._next_id() request = encoder_fn(client_id=self.client_id, correlation_id=requestId, payloads=payloads) failed = False # Send the request, recv the response try: conn.send(requestId, request) if decoder_fn is None: continue try: response = conn.recv(requestId) except ConnectionError as e: log.warning("Could not receive response to request [%s] " "from server %s: %s", binascii.b2a_hex(request), conn, e) failed = True except ConnectionError as e: log.warning("Could not send request [%s] to server %s: %s", binascii.b2a_hex(request), conn, e) failed = True if failed: failed_payloads += payloads self.reset_all_metadata() continue for response in decoder_fn(response): acc[(response.topic, response.partition)] = response if failed_payloads: raise FailedPayloadsError(failed_payloads) # Order the accumulated responses by the original key order return (acc[k] for k in original_keys) if acc else ()
def send_side_effect(reqs, *args, **kwargs): return [FailedPayloadsError(req) for req in reqs]
def _send_consumer_aware_request(self, group, payloads, encoder_fn, decoder_fn): """ Send a list of requests to the consumer coordinator for the group specified using the supplied encode/decode functions. As the payloads that use consumer-aware requests do not contain the group (e.g. OffsetFetchRequest), all payloads must be for a single group. Arguments: group: the name of the consumer group (str) the payloads are for payloads: list of object-like entities with topic (str) and partition (int) attributes; payloads with duplicate topic+partition are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] broker = self._get_coordinator_for_group(group) # Send the list of request payloads and collect the responses and # errors responses = {} requestId = self._next_id() log.debug('Request %s to %s: %s', requestId, broker, payloads) request = encoder_fn(client_id=self.client_id, correlation_id=requestId, payloads=payloads) # Send the request, recv the response try: host, port, afi = get_ip_port_afi(broker.host) conn = self._get_conn(host, broker.port, afi) conn.send(requestId, request) except ConnectionError as e: log.warning( 'ConnectionError attempting to send request %s ' 'to server %s: %s', requestId, broker, e) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) # No exception, try to get response else: # decoder_fn=None signal that the server is expected to not # send a response. This probably only applies to # ProduceRequest w/ acks = 0 if decoder_fn is None: log.debug( 'Request %s does not expect a response ' '(skipping conn.recv)', requestId) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = None return [] try: response = conn.recv(requestId) except ConnectionError as e: log.warning( 'ConnectionError attempting to receive a ' 'response to request %s from server %s: %s', requestId, broker, e) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) else: _resps = [] for payload_response in decoder_fn(response): topic_partition = (payload_response.topic, payload_response.partition) responses[topic_partition] = payload_response _resps.append(payload_response) log.debug('Response %s: %s', requestId, _resps) # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def failed_payloads(payloads): for payload in payloads: topic_partition = (str(payload.topic), payload.partition) responses[(topic_partition)] = FailedPayloadsError(payload)
def _send_broker_aware_request(self, payloads, encoder_fn, decoder_fn): """ Group a list of request payloads by topic+partition and send them to the leader broker for that partition using the supplied encode/decode functions Arguments: payloads: list of object-like entities with a topic (str) and partition (int) attribute; payloads with duplicate topic-partitions are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] # Group the requests by topic+partition brokers_for_payloads = [] payloads_by_broker = collections.defaultdict(list) responses = {} for payload in payloads: try: leader = self._get_leader_for_partition( payload.topic, payload.partition) payloads_by_broker[leader].append(payload) brokers_for_payloads.append(leader) except KafkaUnavailableError as e: log.warning( 'KafkaUnavailableError attempting to send request ' 'on topic %s partition %d', payload.topic, payload.partition) topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) # For each broker, send the list of request payloads # and collect the responses and errors broker_failures = [] # For each KafkaConnection keep the real socket so that we can use # a select to perform unblocking I/O connections_by_socket = {} for broker, payloads in payloads_by_broker.items(): requestId = self._next_id() log.debug('Request %s to %s: %s', requestId, broker, payloads) request = encoder_fn(client_id=self.client_id, correlation_id=requestId, payloads=payloads) # Send the request, recv the response try: conn = self._get_conn(broker.host.decode('utf-8'), broker.port) conn.send(requestId, request) except ConnectionError as e: broker_failures.append(broker) log.warning( 'ConnectionError attempting to send request %s ' 'to server %s: %s', requestId, broker, e) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) # No exception, try to get response else: # decoder_fn=None signal that the server is expected to not # send a response. This probably only applies to # ProduceRequest w/ acks = 0 if decoder_fn is None: log.debug( 'Request %s does not expect a response ' '(skipping conn.recv)', requestId) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = None continue else: connections_by_socket[conn.get_connected_socket()] = ( conn, broker, requestId) conn = None while connections_by_socket: sockets = connections_by_socket.keys() rlist, _, _ = select.select(sockets, [], [], self.timeout) if rlist: conn, broker, requestId = connections_by_socket.pop(rlist[0]) try: response = conn.recv(requestId) except ConnectionError as e: broker_failures.append(broker) log.warning( 'ConnectionError attempting to receive a ' 'response to request %s from server %s: %s', requestId, broker, e) for payload in payloads_by_broker[broker]: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError( payload) else: _resps = [] for payload_response in decoder_fn(response): topic_partition = (payload_response.topic, payload_response.partition) responses[topic_partition] = payload_response _resps.append(payload_response) log.debug('Response %s: %s', requestId, _resps) # If the timeout expires rlist is empty and # all pending requests are considered failed else: for conn, broker, requestId in connections_by_socket.values(): conn.close() broker_failures.append(broker) log.warning( 'Socket timeout error attempting to receive a ' 'response to request %s from server %s', requestId, broker) for payload in payloads_by_broker[broker]: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError( payload) # Connection errors generally mean stale metadata # although sometimes it means incorrect api request # Unfortunately there is no good way to tell the difference # so we'll just reset metadata on all errors to be safe if broker_failures: self.reset_all_metadata() # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def _send_broker_aware_request(self, payloads, encoder_fn, decoder_fn): """ Group a list of request payloads by topic+partition and send them to the leader broker for that partition using the supplied encode/decode functions Arguments: payloads: list of object-like entities with a topic (str) and partition (int) attribute encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ log.debug("Sending Payloads: %s" % payloads) # Group the requests by topic+partition brokers_for_payloads = [] payloads_by_broker = collections.defaultdict(list) for payload in payloads: leader = self._get_leader_for_partition(payload.topic, payload.partition) payloads_by_broker[leader].append(payload) brokers_for_payloads.append(leader) # For each broker, send the list of request payloads # and collect the responses and errors responses_by_broker = collections.defaultdict(list) broker_failures = [] for broker, payloads in payloads_by_broker.items(): conn = self._get_conn(broker.host.decode('utf-8'), broker.port) requestId = self._next_id() request = encoder_fn(client_id=self.client_id, correlation_id=requestId, payloads=payloads) # Send the request, recv the response try: conn.send(requestId, request) except ConnectionError as e: broker_failures.append(broker) log.warning("Could not send request [%s] to server %s: %s", binascii.b2a_hex(request), conn, e) for payload in payloads: responses_by_broker[broker].append( FailedPayloadsError(payload)) # No exception, try to get response else: # decoder_fn=None signal that the server is expected to not # send a response. This probably only applies to # ProduceRequest w/ acks = 0 if decoder_fn is None: for payload in payloads: responses_by_broker[broker].append(None) continue try: response = conn.recv(requestId) except ConnectionError as e: broker_failures.append(broker) log.warning( "Could not receive response to request [%s] " "from server %s: %s", binascii.b2a_hex(request), conn, e) for payload in payloads: responses_by_broker[broker].append( FailedPayloadsError(payload)) else: for payload_response in decoder_fn(response): responses_by_broker[broker].append(payload_response) # Connection errors generally mean stale metadata # although sometimes it means incorrect api request # Unfortunately there is no good way to tell the difference # so we'll just reset metadata on all errors to be safe if broker_failures: self.reset_all_metadata() # Return responses in the same order as provided responses_by_payload = [ responses_by_broker[broker].pop(0) for broker in brokers_for_payloads ] log.debug('Responses: %s' % responses_by_payload) return responses_by_payload