def test_call_actions_parallel_with_transport_errors_caught(self):
original_send = self.client.send_request
side_effect_context = {'call': 0}
error = MessageSendError('Hello!')
def side_effect(*args, **kwargs):
side_effect_context['call'] += 1
if side_effect_context['call'] == 2:
raise error
return original_send(*args, **kwargs)
with mock.patch.object(self.client, 'send_request') as mock_send_request:
mock_send_request.side_effect = side_effect
action_responses = self.client.call_actions_parallel(
'error_service',
[
ActionRequest(action='okay_action'),
ActionRequest(action='job_error'),
ActionRequest(action='okay_action'),
],
timeout=2,
catch_transport_errors=True,
continue_on_error=True,
)
self.assertIsNotNone(action_responses)
action_responses = list(action_responses)
self.assertEqual(3, len(action_responses))
self.assertEqual({'no_error': True}, action_responses[0].body)
self.assertIs(error, action_responses[1])
self.assertEqual({'no_error': True}, action_responses[2].body)
def send_request_message(self, request_id, meta, body, message_expiry_in_seconds=None):
raise MessageSendError('The message failed to send')
def send_message(self,
queue_name,
request_id,
meta,
body,
message_expiry_in_seconds=None):
"""
Send a message to the specified queue in Redis.
:param queue_name: The name of the queue to which to send the message
:type queue_name: union(str, unicode)
:param request_id: The message's request ID
:type request_id: int
:param meta: The message meta information, if any (should be an empty dict if no metadata)
:type meta: dict
:param body: The message body (should be a dict)
:type body: dict
:param message_expiry_in_seconds: The optional message expiry, which defaults to the setting with the same name
:type message_expiry_in_seconds: int
:raise: InvalidMessageError, MessageTooLarge, MessageSendError
"""
if request_id is None:
raise InvalidMessageError('No request ID')
if message_expiry_in_seconds:
message_expiry = time.time() + message_expiry_in_seconds
redis_expiry = message_expiry_in_seconds + 10
else:
message_expiry = time.time() + self.message_expiry_in_seconds
redis_expiry = self.message_expiry_in_seconds
meta['__expiry__'] = message_expiry
message = {'request_id': request_id, 'meta': meta, 'body': body}
with self._get_timer('send.serialize'):
serialized_message = self.serializer.dict_to_blob(message)
message_size_in_bytes = len(serialized_message)
if message_size_in_bytes > self.maximum_message_size_in_bytes:
self._get_counter('send.error.message_too_large').increment()
raise MessageTooLarge(message_size_in_bytes)
elif self.log_messages_larger_than_bytes and message_size_in_bytes > self.log_messages_larger_than_bytes:
_oversized_message_logger.warning(
'Oversized message sent for PySOA service {}'.format(
self.service_name),
extra={
'data': {
'message': RecursivelyCensoredDictWrapper(message),
'serialized_length_in_bytes': message_size_in_bytes,
'threshold': self.log_messages_larger_than_bytes,
}
},
)
queue_key = self.QUEUE_NAME_PREFIX + queue_name
# Try at least once, up to queue_full_retries times, then error
for i in range(-1, self.queue_full_retries):
if i >= 0:
time.sleep((2**i + random.random()) /
self.EXPONENTIAL_BACK_OFF_FACTOR)
self._get_counter('send.queue_full_retry').increment()
self._get_counter(
'send.queue_full_retry.retry_{}'.format(i +
1)).increment()
try:
with self._get_timer('send.get_redis_connection'):
connection = self.backend_layer.get_connection(queue_key)
with self._get_timer('send.send_message_to_redis_queue'):
self.backend_layer.send_message_to_queue(
queue_key=queue_key,
message=serialized_message,
expiry=redis_expiry,
capacity=self.queue_capacity,
connection=connection,
)
return
except redis.exceptions.ResponseError as e:
# The Lua script handles capacity checking and sends the "full" error back
if e.args[0] == 'queue full':
continue
self._get_counter('send.error.response').increment()
raise MessageSendError(
'Redis error sending message for service {}'.format(
self.service_name), *e.args)
except CannotGetConnectionError as e:
self._get_counter('send.error.connection').increment()
raise MessageSendError('Cannot get connection: {}'.format(
e.args[0]))
except Exception as e:
self._get_counter('send.error.unknown').increment()
raise MessageSendError(
'Unknown error sending message for service {}'.format(
self.service_name), six.text_type(type(e).__name__),
*e.args)
self._get_counter('send.error.redis_queue_full').increment()
raise MessageSendError(
'Redis queue {queue_name} was full after {retries} retries'.format(
queue_name=queue_name,
retries=self.queue_full_retries,
))
def send_message(self, request_id, meta, body, message_expiry_in_seconds=None):
protocol_key = meta.get('protocol_key')
stream_id = meta.get('stream_id')
if request_id is None:
raise InvalidMessageError('No request ID')
if message_expiry_in_seconds:
message_expiry = time.time() + message_expiry_in_seconds
else:
message_expiry = time.time() + self.message_expiry_in_seconds
meta['__expiry__'] = message_expiry
message = {'request_id': request_id, 'meta': meta, 'body': body}
with self._get_timer('send.serialize'):
serializer = self.default_serializer
if 'serializer' in meta:
# TODO: Breaking change: Assume a MIME type is always specified. This should not be done until all
# TODO servers and clients have Step 2 code. This will be a Step 3 breaking change.
serializer = meta.pop('serializer')
serialized_message = (
'content-type:{};'.format(serializer.mime_type).encode('utf-8') + serializer.dict_to_blob(message)
)
message_size_in_bytes = len(serialized_message)
response_headers = [
(':status', '200'),
('content-type', 'application/json'),
('content-length', str(message_size_in_bytes)),
('server', 'pysoa-h2'),
]
if self.log_messages_larger_than_bytes and message_size_in_bytes > self.log_messages_larger_than_bytes:
_oversized_message_logger.warning(
'Oversized message sent for PySOA service {}'.format(self.service_name),
extra={'data': {
'message': RecursivelyCensoredDictWrapper(message),
'serialized_length_in_bytes': message_size_in_bytes,
'threshold': self.log_messages_larger_than_bytes,
}},
)
for i in range(-1, self.queue_full_retries):
if i >= 0:
time.sleep((2 ** i + random.random()) / self.EXPONENTIAL_BACK_OFF_FACTOR)
self._get_counter('send.responses_queue_full_retry').increment()
self._get_counter('send.responses_queue_full_retry.retry_{}'.format(i + 1)).increment()
try:
with self._get_timer('send.send_message_response_http2_queue'):
self.responses_queue.put((
protocol_key,
stream_id,
request_id,
serialized_message,
response_headers,
), timeout=0)
return
except six.moves.queue.Full:
continue
except Exception as e:
self._get_counter('send.error.unknown').increment()
raise MessageSendError(
'Unknown error sending message for service {}'.format(self.service_name),
six.text_type(type(e).__name__),
*e.args
)
self._get_counter('send.error.responses_queue_full').increment()
raise MessageSendError(
'Http2 responses queue was full after {retries} retries'.format(
retries=self.queue_full_retries,
)
)
def send_message(self, queue_name, request_id, meta, body):
"""
Send a message to the specified queue in Redis.
:param queue_name: The name of the queue to which to send the message
:param request_id: The message's request ID
:param meta: The meta information, if any (should be an empty dict if no metadata)
:param body: The message body (should be a dict)
:raise
"""
if request_id is None:
raise InvalidMessageError('No request ID')
meta['__expiry__'] = self._get_message_expiry()
message = {'request_id': request_id, 'meta': meta, 'body': body}
with self._get_timer('send.serialize'):
serialized_message = self.serializer.dict_to_blob(message)
if len(serialized_message) > self.MAXIMUM_MESSAGE_BYTES:
self._get_counter('send.error.message_too_large').increment()
raise MessageTooLarge()
queue_key = self.QUEUE_NAME_PREFIX + queue_name
# Try at least once, up to queue_full_retries times, then error
for i in range(-1, self.queue_full_retries):
if i >= 0:
time.sleep((2 ** i + random.random()) / self.EXPONENTIAL_BACK_OFF_FACTOR)
self._get_counter('send.queue_full_retry').increment()
self._get_counter('send.queue_full_retry.retry_{}'.format(i + 1)).increment()
try:
with self._get_timer('send.get_redis_connection'):
connection = self.backend_layer.get_connection(queue_key)
with self._get_timer('send.send_message_to_redis_queue'):
self.backend_layer.send_message_to_queue(
queue_key=queue_key,
message=serialized_message,
expiry=self.message_expiry_in_seconds,
capacity=self.queue_capacity,
connection=connection,
)
return
except redis.exceptions.ResponseError as e:
# The Lua script handles capacity checking and sends the "full" error back
if e.args[0] == 'queue full':
continue
self._get_counter('send.error.response').increment()
raise MessageSendError('Redis error sending message for service {}'.format(self.service_name), *e.args)
except CannotGetConnectionError as e:
self._get_counter('send.error.connection').increment()
raise MessageSendError('Cannot get connection: {}'.format(e.args[0]))
except Exception as e:
self._get_counter('send.error.unknown').increment()
raise MessageSendError(
'Unknown error sending message for service {}'.format(self.service_name),
six.text_type(type(e).__name__),
*e.args
)
self._get_counter('send.error.redis_queue_full').increment()
raise MessageSendError(
'Redis queue {queue_name} was full after {retries} retries'.format(
queue_name=queue_name,
retries=self.queue_full_retries,
)
)