class RedisServerTransport(ServerTransport):
def __init__(self, service_name, metrics, **kwargs):
super(RedisServerTransport, self).__init__(service_name, metrics)
self._receive_queue_name = make_redis_queue_name(service_name)
self.core = RedisTransportCore(service_name=service_name,
metrics=metrics,
metrics_prefix='server',
**kwargs)
def receive_request_message(self):
timer = self.metrics.timer('server.transport.redis_gateway.receive')
timer.start()
stop_timer = True
try:
return self.core.receive_message(self._receive_queue_name)
except MessageReceiveTimeout:
stop_timer = False
raise
finally:
if stop_timer:
timer.stop()
def send_response_message(self, request_id, meta, body):
try:
queue_name = meta['reply_to']
except KeyError:
self.metrics.counter(
'server.transport.redis_gateway.send.error.missing_reply_queue'
)
raise InvalidMessageError('Missing reply queue name')
with self.metrics.timer('server.transport.redis_gateway.send'):
self.core.send_message(queue_name, request_id, meta, body)
def __init__(self, service_name, metrics, **kwargs):
"""
In addition to the two named positional arguments, this constructor expects keyword arguments abiding by the
Redis transport settings schema.
:param service_name: The name of the service to which this transport will send requests (and from which it will
receive responses)
:type service_name: union[str, unicode]
:param metrics: The optional metrics recorder
:type metrics: MetricsRecorder
"""
super(RedisClientTransport, self).__init__(service_name, metrics)
if 'maximum_message_size_in_bytes' not in kwargs:
kwargs['maximum_message_size_in_bytes'] = DEFAULT_MAXIMUM_MESSAGE_BYTES_CLIENT
self.client_id = uuid.uuid4().hex
self._send_queue_name = make_redis_queue_name(service_name)
self._receive_queue_name = '{send_queue_name}.{client_id}{response_queue_specifier}'.format(
send_queue_name=self._send_queue_name,
client_id=self.client_id,
response_queue_specifier=BaseRedisClient.RESPONSE_QUEUE_SPECIFIER,
)
self._requests_outstanding = 0
self.core = RedisTransportCore(service_name=service_name, metrics=metrics, metrics_prefix='client', **kwargs)
def __init__(self, service_name, metrics, **kwargs):
super(RedisServerTransport, self).__init__(service_name, metrics)
self._receive_queue_name = make_redis_queue_name(service_name)
self.core = RedisTransportCore(service_name=service_name,
metrics=metrics,
metrics_prefix='server',
**kwargs)
def test_cannot_get_connection_error_on_receive(self, mock_standard, mock_sentinel):
core = RedisTransportCore(backend_type=REDIS_BACKEND_TYPE_STANDARD)
mock_standard.return_value.get_connection.side_effect = CannotGetConnectionError('This is another error')
with self.assertRaises(MessageReceiveError) as error_context:
core.receive_message('your_queue')
self.assertEquals('Cannot get connection: This is another error', error_context.exception.args[0])
self.assertFalse(mock_sentinel.called)
mock_standard.return_value.get_connection.assert_called_once_with('pysoa:your_queue')
class RedisServerTransport(ServerTransport):
def __init__(self, service_name, metrics, **kwargs):
"""
In addition to the two named positional arguments, this constructor expects keyword arguments abiding by the
Redis transport settings schema.
:param service_name: The name of the service for which this transport will receive requests and send responses
:type service_name: union[str, unicode]
:param metrics: The optional metrics recorder
:type metrics: MetricsRecorder
"""
super(RedisServerTransport, self).__init__(service_name, metrics)
if 'maximum_message_size_in_bytes' not in kwargs:
kwargs[
'maximum_message_size_in_bytes'] = DEFAULT_MAXIMUM_MESSAGE_BYTES_SERVER
self._receive_queue_name = make_redis_queue_name(service_name)
self.core = RedisTransportCore(service_name=service_name,
metrics=metrics,
metrics_prefix='server',
**kwargs)
def receive_request_message(self):
timer = self.metrics.timer('server.transport.redis_gateway.receive',
resolution=TimerResolution.MICROSECONDS)
timer.start()
stop_timer = True
try:
return self.core.receive_message(self._receive_queue_name)
except MessageReceiveTimeout:
stop_timer = False
raise
finally:
if stop_timer:
timer.stop()
def send_response_message(self, request_id, meta, body):
try:
queue_name = meta['reply_to']
except KeyError:
self.metrics.counter(
'server.transport.redis_gateway.send.error.missing_reply_queue'
)
raise InvalidMessageError('Missing reply queue name')
with self.metrics.timer('server.transport.redis_gateway.send',
resolution=TimerResolution.MICROSECONDS):
self.core.send_message(queue_name, request_id, meta, body)
def test_standard_client_created(self, mock_standard, mock_sentinel):
core = RedisTransportCore(
service_name='example',
backend_type=REDIS_BACKEND_TYPE_STANDARD,
backend_layer_kwargs={
'hosts': [('localhost', 6379), 'far_away_host'],
'redis_db': 2,
'redis_port': 1098,
},
message_expiry_in_seconds=30,
queue_capacity=100,
queue_full_retries=7,
receive_timeout_in_seconds=10,
serializer_config={'object': MockSerializer, 'kwargs': {'kwarg1': 'hello'}},
)
core.backend_layer.anything()
self.assertEqual('example', core.service_name)
self.assertEqual(30, core.message_expiry_in_seconds)
self.assertEqual(100, core.queue_capacity)
self.assertEqual(7, core.queue_full_retries)
self.assertEqual(10, core.receive_timeout_in_seconds)
self.assertIsInstance(core.serializer, MockSerializer)
self.assertEqual('hello', core.serializer.kwarg1)
mock_standard.assert_called_once_with(
hosts=[('localhost', 6379), ('far_away_host', 1098)],
connection_kwargs={'db': 2},
)
mock_standard.return_value.anything.assert_called_once_with()
self.assertFalse(mock_sentinel.called)
def __init__(self, service_name, metrics, **kwargs):
"""
In addition to the two named positional arguments, this constructor expects keyword arguments abiding by the
Redis transport settings schema.
:param service_name: The name of the service for which this transport will receive requests and send responses
:type service_name: union[str, unicode]
:param metrics: The optional metrics recorder
:type metrics: MetricsRecorder
"""
super(RedisServerTransport, self).__init__(service_name, metrics)
if 'maximum_message_size_in_bytes' not in kwargs:
kwargs['maximum_message_size_in_bytes'] = DEFAULT_MAXIMUM_MESSAGE_BYTES_SERVER
self._receive_queue_name = make_redis_queue_name(service_name)
self.core = RedisTransportCore(service_name=service_name, metrics=metrics, metrics_prefix='server', **kwargs)
def test_standard_client_created_with_defaults(self, mock_standard, mock_sentinel):
core = RedisTransportCore(backend_type=REDIS_BACKEND_TYPE_STANDARD)
core.backend_layer.anything()
self.assertEqual(60, core.message_expiry_in_seconds)
self.assertEqual(10000, core.queue_capacity)
self.assertEqual(10, core.queue_full_retries)
self.assertEqual(5, core.receive_timeout_in_seconds)
self.assertIsInstance(core.serializer, MsgpackSerializer)
mock_standard.assert_called_once_with()
mock_standard.return_value.anything.assert_called_once_with()
self.assertFalse(mock_sentinel.called)
def test_sentinel_client_created(self, mock_standard, mock_sentinel):
core = RedisTransportCore(
backend_type=REDIS_BACKEND_TYPE_SENTINEL,
backend_layer_kwargs={
'connection_kwargs': {
'hello': 'world'
},
'hosts': [('another_host', 6379)],
'redis_db': 5,
'redis_port': 1098,
'sentinel_refresh_interval': 13,
'sentinel_services': ['svc1', 'svc2', 'svc3'],
'sentinel_failover_retries': 5,
},
message_expiry_in_seconds=45,
queue_capacity=7500,
queue_full_retries=4,
receive_timeout_in_seconds=6,
default_serializer_config={
'object': MockSerializer,
'kwargs': {
'kwarg2': 'goodbye'
}
},
)
core.backend_layer.anything()
self.assertEqual(45, core.message_expiry_in_seconds)
self.assertEqual(7500, core.queue_capacity)
self.assertEqual(4, core.queue_full_retries)
self.assertEqual(6, core.receive_timeout_in_seconds)
self.assertIsInstance(core.default_serializer, MockSerializer)
self.assertEqual('goodbye', core.default_serializer.kwarg2)
mock_sentinel.assert_called_once_with(
hosts=[('another_host', 6379)],
connection_kwargs={
'db': 5,
'hello': 'world'
},
sentinel_refresh_interval=13,
sentinel_services=['svc1', 'svc2', 'svc3'],
sentinel_failover_retries=5,
)
mock_sentinel.return_value.anything.assert_called_once_with()
self.assertFalse(mock_standard.called)
def test_invalid_backend_type(self):
with self.assertRaises(ValueError):
RedisTransportCore(backend_type='hello')
def _get_core(**kwargs):
return RedisTransportCore(backend_type=REDIS_BACKEND_TYPE_STANDARD,
**kwargs)
class RedisClientTransport(ClientTransport):
def __init__(self, service_name, metrics, **kwargs):
"""
In addition to the two named positional arguments, this constructor expects keyword arguments abiding by the
Redis transport settings schema.
:param service_name: The name of the service to which this transport will send requests (and from which it will
receive responses)
:type service_name: union[str, unicode]
:param metrics: The optional metrics recorder
:type metrics: MetricsRecorder
"""
super(RedisClientTransport, self).__init__(service_name, metrics)
if 'maximum_message_size_in_bytes' not in kwargs:
kwargs['maximum_message_size_in_bytes'] = DEFAULT_MAXIMUM_MESSAGE_BYTES_CLIENT
self.client_id = uuid.uuid4().hex
self._send_queue_name = make_redis_queue_name(service_name)
self._receive_queue_name = '{send_queue_name}.{client_id}{response_queue_specifier}'.format(
send_queue_name=self._send_queue_name,
client_id=self.client_id,
response_queue_specifier=BaseRedisClient.RESPONSE_QUEUE_SPECIFIER,
)
self._requests_outstanding = 0
self.core = RedisTransportCore(service_name=service_name, metrics=metrics, metrics_prefix='client', **kwargs)
@property
def requests_outstanding(self):
"""
Indicates the number of requests currently outstanding, which still need to be received. If this value is less
than 1, calling `receive_response_message` will result in a return value of `(None, None, None)` instead of
raising a `MessageReceiveTimeout`.
"""
return self._requests_outstanding
def send_request_message(self, request_id, meta, body, message_expiry_in_seconds=None):
self._requests_outstanding += 1
meta['reply_to'] = '{receive_queue_name}{thread_id}'.format(
receive_queue_name=self._receive_queue_name,
thread_id=get_hex_thread_id(),
)
with self.metrics.timer('client.transport.redis_gateway.send', resolution=TimerResolution.MICROSECONDS):
self.core.send_message(self._send_queue_name, request_id, meta, body, message_expiry_in_seconds)
def receive_response_message(self, receive_timeout_in_seconds=None):
if self._requests_outstanding > 0:
with self.metrics.timer('client.transport.redis_gateway.receive', resolution=TimerResolution.MICROSECONDS):
try:
request_id, meta, response = self.core.receive_message(
'{receive_queue_name}{thread_id}'.format(
receive_queue_name=self._receive_queue_name,
thread_id=get_hex_thread_id(),
),
receive_timeout_in_seconds,
)
except MessageReceiveTimeout:
self.metrics.counter('client.transport.redis_gateway.receive.error.timeout').increment()
raise
self._requests_outstanding -= 1
return request_id, meta, response
else:
# This tells Client.get_all_responses to stop waiting for more.
return None, None, None