class RedisServerTransport(ServerTransport): def __init__(self, service_name, metrics, **kwargs): super(RedisServerTransport, self).__init__(service_name, metrics) self._receive_queue_name = make_redis_queue_name(service_name) self.core = RedisTransportCore(service_name=service_name, metrics=metrics, metrics_prefix='server', **kwargs) def receive_request_message(self): timer = self.metrics.timer('server.transport.redis_gateway.receive') timer.start() stop_timer = True try: return self.core.receive_message(self._receive_queue_name) except MessageReceiveTimeout: stop_timer = False raise finally: if stop_timer: timer.stop() def send_response_message(self, request_id, meta, body): try: queue_name = meta['reply_to'] except KeyError: self.metrics.counter( 'server.transport.redis_gateway.send.error.missing_reply_queue' ) raise InvalidMessageError('Missing reply queue name') with self.metrics.timer('server.transport.redis_gateway.send'): self.core.send_message(queue_name, request_id, meta, body)
def __init__(self, service_name, metrics, **kwargs): """ In addition to the two named positional arguments, this constructor expects keyword arguments abiding by the Redis transport settings schema. :param service_name: The name of the service to which this transport will send requests (and from which it will receive responses) :type service_name: union[str, unicode] :param metrics: The optional metrics recorder :type metrics: MetricsRecorder """ super(RedisClientTransport, self).__init__(service_name, metrics) if 'maximum_message_size_in_bytes' not in kwargs: kwargs['maximum_message_size_in_bytes'] = DEFAULT_MAXIMUM_MESSAGE_BYTES_CLIENT self.client_id = uuid.uuid4().hex self._send_queue_name = make_redis_queue_name(service_name) self._receive_queue_name = '{send_queue_name}.{client_id}{response_queue_specifier}'.format( send_queue_name=self._send_queue_name, client_id=self.client_id, response_queue_specifier=BaseRedisClient.RESPONSE_QUEUE_SPECIFIER, ) self._requests_outstanding = 0 self.core = RedisTransportCore(service_name=service_name, metrics=metrics, metrics_prefix='client', **kwargs)
def __init__(self, service_name, metrics, **kwargs): super(RedisServerTransport, self).__init__(service_name, metrics) self._receive_queue_name = make_redis_queue_name(service_name) self.core = RedisTransportCore(service_name=service_name, metrics=metrics, metrics_prefix='server', **kwargs)
def test_cannot_get_connection_error_on_receive(self, mock_standard, mock_sentinel): core = RedisTransportCore(backend_type=REDIS_BACKEND_TYPE_STANDARD) mock_standard.return_value.get_connection.side_effect = CannotGetConnectionError('This is another error') with self.assertRaises(MessageReceiveError) as error_context: core.receive_message('your_queue') self.assertEquals('Cannot get connection: This is another error', error_context.exception.args[0]) self.assertFalse(mock_sentinel.called) mock_standard.return_value.get_connection.assert_called_once_with('pysoa:your_queue')
class RedisServerTransport(ServerTransport): def __init__(self, service_name, metrics, **kwargs): """ In addition to the two named positional arguments, this constructor expects keyword arguments abiding by the Redis transport settings schema. :param service_name: The name of the service for which this transport will receive requests and send responses :type service_name: union[str, unicode] :param metrics: The optional metrics recorder :type metrics: MetricsRecorder """ super(RedisServerTransport, self).__init__(service_name, metrics) if 'maximum_message_size_in_bytes' not in kwargs: kwargs[ 'maximum_message_size_in_bytes'] = DEFAULT_MAXIMUM_MESSAGE_BYTES_SERVER self._receive_queue_name = make_redis_queue_name(service_name) self.core = RedisTransportCore(service_name=service_name, metrics=metrics, metrics_prefix='server', **kwargs) def receive_request_message(self): timer = self.metrics.timer('server.transport.redis_gateway.receive', resolution=TimerResolution.MICROSECONDS) timer.start() stop_timer = True try: return self.core.receive_message(self._receive_queue_name) except MessageReceiveTimeout: stop_timer = False raise finally: if stop_timer: timer.stop() def send_response_message(self, request_id, meta, body): try: queue_name = meta['reply_to'] except KeyError: self.metrics.counter( 'server.transport.redis_gateway.send.error.missing_reply_queue' ) raise InvalidMessageError('Missing reply queue name') with self.metrics.timer('server.transport.redis_gateway.send', resolution=TimerResolution.MICROSECONDS): self.core.send_message(queue_name, request_id, meta, body)
def test_standard_client_created(self, mock_standard, mock_sentinel): core = RedisTransportCore( service_name='example', backend_type=REDIS_BACKEND_TYPE_STANDARD, backend_layer_kwargs={ 'hosts': [('localhost', 6379), 'far_away_host'], 'redis_db': 2, 'redis_port': 1098, }, message_expiry_in_seconds=30, queue_capacity=100, queue_full_retries=7, receive_timeout_in_seconds=10, serializer_config={'object': MockSerializer, 'kwargs': {'kwarg1': 'hello'}}, ) core.backend_layer.anything() self.assertEqual('example', core.service_name) self.assertEqual(30, core.message_expiry_in_seconds) self.assertEqual(100, core.queue_capacity) self.assertEqual(7, core.queue_full_retries) self.assertEqual(10, core.receive_timeout_in_seconds) self.assertIsInstance(core.serializer, MockSerializer) self.assertEqual('hello', core.serializer.kwarg1) mock_standard.assert_called_once_with( hosts=[('localhost', 6379), ('far_away_host', 1098)], connection_kwargs={'db': 2}, ) mock_standard.return_value.anything.assert_called_once_with() self.assertFalse(mock_sentinel.called)
def __init__(self, service_name, metrics, **kwargs): """ In addition to the two named positional arguments, this constructor expects keyword arguments abiding by the Redis transport settings schema. :param service_name: The name of the service for which this transport will receive requests and send responses :type service_name: union[str, unicode] :param metrics: The optional metrics recorder :type metrics: MetricsRecorder """ super(RedisServerTransport, self).__init__(service_name, metrics) if 'maximum_message_size_in_bytes' not in kwargs: kwargs['maximum_message_size_in_bytes'] = DEFAULT_MAXIMUM_MESSAGE_BYTES_SERVER self._receive_queue_name = make_redis_queue_name(service_name) self.core = RedisTransportCore(service_name=service_name, metrics=metrics, metrics_prefix='server', **kwargs)
def test_standard_client_created_with_defaults(self, mock_standard, mock_sentinel): core = RedisTransportCore(backend_type=REDIS_BACKEND_TYPE_STANDARD) core.backend_layer.anything() self.assertEqual(60, core.message_expiry_in_seconds) self.assertEqual(10000, core.queue_capacity) self.assertEqual(10, core.queue_full_retries) self.assertEqual(5, core.receive_timeout_in_seconds) self.assertIsInstance(core.serializer, MsgpackSerializer) mock_standard.assert_called_once_with() mock_standard.return_value.anything.assert_called_once_with() self.assertFalse(mock_sentinel.called)
def test_sentinel_client_created(self, mock_standard, mock_sentinel): core = RedisTransportCore( backend_type=REDIS_BACKEND_TYPE_SENTINEL, backend_layer_kwargs={ 'connection_kwargs': { 'hello': 'world' }, 'hosts': [('another_host', 6379)], 'redis_db': 5, 'redis_port': 1098, 'sentinel_refresh_interval': 13, 'sentinel_services': ['svc1', 'svc2', 'svc3'], 'sentinel_failover_retries': 5, }, message_expiry_in_seconds=45, queue_capacity=7500, queue_full_retries=4, receive_timeout_in_seconds=6, default_serializer_config={ 'object': MockSerializer, 'kwargs': { 'kwarg2': 'goodbye' } }, ) core.backend_layer.anything() self.assertEqual(45, core.message_expiry_in_seconds) self.assertEqual(7500, core.queue_capacity) self.assertEqual(4, core.queue_full_retries) self.assertEqual(6, core.receive_timeout_in_seconds) self.assertIsInstance(core.default_serializer, MockSerializer) self.assertEqual('goodbye', core.default_serializer.kwarg2) mock_sentinel.assert_called_once_with( hosts=[('another_host', 6379)], connection_kwargs={ 'db': 5, 'hello': 'world' }, sentinel_refresh_interval=13, sentinel_services=['svc1', 'svc2', 'svc3'], sentinel_failover_retries=5, ) mock_sentinel.return_value.anything.assert_called_once_with() self.assertFalse(mock_standard.called)
def test_invalid_backend_type(self): with self.assertRaises(ValueError): RedisTransportCore(backend_type='hello')
def _get_core(**kwargs): return RedisTransportCore(backend_type=REDIS_BACKEND_TYPE_STANDARD, **kwargs)
class RedisClientTransport(ClientTransport): def __init__(self, service_name, metrics, **kwargs): """ In addition to the two named positional arguments, this constructor expects keyword arguments abiding by the Redis transport settings schema. :param service_name: The name of the service to which this transport will send requests (and from which it will receive responses) :type service_name: union[str, unicode] :param metrics: The optional metrics recorder :type metrics: MetricsRecorder """ super(RedisClientTransport, self).__init__(service_name, metrics) if 'maximum_message_size_in_bytes' not in kwargs: kwargs['maximum_message_size_in_bytes'] = DEFAULT_MAXIMUM_MESSAGE_BYTES_CLIENT self.client_id = uuid.uuid4().hex self._send_queue_name = make_redis_queue_name(service_name) self._receive_queue_name = '{send_queue_name}.{client_id}{response_queue_specifier}'.format( send_queue_name=self._send_queue_name, client_id=self.client_id, response_queue_specifier=BaseRedisClient.RESPONSE_QUEUE_SPECIFIER, ) self._requests_outstanding = 0 self.core = RedisTransportCore(service_name=service_name, metrics=metrics, metrics_prefix='client', **kwargs) @property def requests_outstanding(self): """ Indicates the number of requests currently outstanding, which still need to be received. If this value is less than 1, calling `receive_response_message` will result in a return value of `(None, None, None)` instead of raising a `MessageReceiveTimeout`. """ return self._requests_outstanding def send_request_message(self, request_id, meta, body, message_expiry_in_seconds=None): self._requests_outstanding += 1 meta['reply_to'] = '{receive_queue_name}{thread_id}'.format( receive_queue_name=self._receive_queue_name, thread_id=get_hex_thread_id(), ) with self.metrics.timer('client.transport.redis_gateway.send', resolution=TimerResolution.MICROSECONDS): self.core.send_message(self._send_queue_name, request_id, meta, body, message_expiry_in_seconds) def receive_response_message(self, receive_timeout_in_seconds=None): if self._requests_outstanding > 0: with self.metrics.timer('client.transport.redis_gateway.receive', resolution=TimerResolution.MICROSECONDS): try: request_id, meta, response = self.core.receive_message( '{receive_queue_name}{thread_id}'.format( receive_queue_name=self._receive_queue_name, thread_id=get_hex_thread_id(), ), receive_timeout_in_seconds, ) except MessageReceiveTimeout: self.metrics.counter('client.transport.redis_gateway.receive.error.timeout').increment() raise self._requests_outstanding -= 1 return request_id, meta, response else: # This tells Client.get_all_responses to stop waiting for more. return None, None, None