def test_nameko_services_can_subscribe_to_other_nameko_services( runner_factory, config_path, router ): config = { WAMP_CONFIG_KEY: { 'config_path': config_path, } } runner = runner_factory(config, WampSubscriber, WampPublisher) runner.start() container = get_container(runner, WampSubscriber) wait_for_subscriptions(container, number_of_subscriptions=2) assert WampSubscriber.messages == [] container = get_container(runner, WampPublisher) wait_for_registrations(container, number_of_registrations=1) with entrypoint_hook(container, "publish_foo") as entrypoint: entrypoint() def waiting_for_the_message(): assert len(WampSubscriber.messages) == 1 assert WampSubscriber.messages == [ ((u'hello foo',), {}) ] assert_stops_raising(waiting_for_the_message) WampSubscriber.messages = []
def test_reconnect_on_socket_error(rabbit_config, mock_container): container = mock_container container.shared_extensions = {} container.config = rabbit_config container.max_workers = 1 container.spawn_managed_thread = spawn_managed_thread connection_revived = Mock() queue_consumer = QueueConsumer().bind(container) queue_consumer.setup() queue_consumer.on_connection_revived = connection_revived handler = MessageHandler() queue_consumer.register_provider(handler) queue_consumer.start() with patch.object(Connection, "drain_events", autospec=True) as drain_events: drain_events.side_effect = socket.error("test-error") def check_reconnected(): assert connection_revived.call_count > 1 assert_stops_raising(check_reconnected) queue_consumer.unregister_provider(handler) queue_consumer.stop()
def test_reconnect_on_socket_error(): container = Mock() container.config = {AMQP_URI_CONFIG_KEY: None} container.max_workers = 1 container.spawn_managed_thread = spawn_thread connection_revived = Mock() queue_consumer = QueueConsumer() queue_consumer.on_connection_revived = connection_revived queue_consumer.bind("queue_consumer", container) handler = MessageHandler() queue_consumer.register_provider(handler) queue_consumer.start() with patch.object(Connection, 'drain_events', autospec=True) as drain_events: drain_events.side_effect = socket.error('test-error') def check_reconnected(): assert connection_revived.call_count > 1 assert_stops_raising(check_reconnected)
def test_reconnect_on_socket_error(): container = Mock() container.config = {AMQP_URI_CONFIG_KEY: None} container.max_workers = 1 container.spawn_managed_thread = spawn_thread connection_revived = Mock() queue_consumer = QueueConsumer() queue_consumer.on_connection_revived = connection_revived queue_consumer.bind("queue_consumer", container) handler = MessageHandler() queue_consumer.register_provider(handler) queue_consumer.start() with patch.object( Connection, 'drain_events', autospec=True) as drain_events: drain_events.side_effect = socket.error('test-error') def check_reconnected(): assert connection_revived.call_count > 1 assert_stops_raising(check_reconnected)
def test_reconnect_on_socket_error(rabbit_config, mock_container): container = mock_container container.shared_extensions = {} container.config = rabbit_config container.max_workers = 1 container.spawn_managed_thread = spawn_managed_thread connection_revived = Mock() queue_consumer = QueueConsumer().bind(container) queue_consumer.setup() queue_consumer.on_connection_revived = connection_revived handler = MessageHandler() queue_consumer.register_provider(handler) queue_consumer.start() with patch.object( Connection, 'drain_events', autospec=True) as drain_events: drain_events.side_effect = socket.error('test-error') def check_reconnected(): assert connection_revived.call_count > 1 assert_stops_raising(check_reconnected) queue_consumer.unregister_provider(handler) queue_consumer.stop()
def test_nameko_service_can_subscribe_to_wamp_topic( runner_factory, router, config_path ): wamp_client = Client(router=router) config = { WAMP_CONFIG_KEY: { 'config_path': config_path, } } runner = runner_factory(config, WampSubscriber) runner.start() container = get_container(runner, WampSubscriber) with wamp_client as client: wait_for_session(client) wait_for_subscriptions(container, number_of_subscriptions=2) client.publish(topic="foo", message="hello foo") client.publish(topic="bar", message="hello bar") def waiting_for_the_message(): assert len(WampSubscriber.messages) == 2 assert sorted(WampSubscriber.messages) == sorted([ ((u'hello foo',), {}), ((u'hello bar',), {}) ]) assert_stops_raising(waiting_for_the_message) WampSubscriber.messages = []
def test_service_disconnect_with_active_async_worker( container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active async worker (e.g. event handler). """ container = container_factory(ExampleService, rabbit_config) container.start() # get the service's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 queue_consumer_conn = connections[0]['name'] # disconnect the service's queue consumer while it's running the worker eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn) # dispatch an event data = uuid.uuid4().hex dispatch = event_dispatcher(rabbit_config) dispatch('srcservice', 'exampleevent', data) # `handle` will have been called twice with the same the `data`, because # rabbit will have redelivered the un-ack'd message from the first call def event_handled_twice(): assert handle_called.call_args_list == [call(data), call(data)] assert_stops_raising(event_handled_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert queue_consumer_conn not in [conn['name'] for conn in connections]
def test_service_disconnect_with_active_async_worker(container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active async worker (e.g. event handler). """ container = container_factory(ExampleService, rabbit_config) container.start() # get the service's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 queue_consumer_conn = connections[0]['name'] # disconnect the service's queue consumer while it's running the worker eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn) # dispatch an event data = uuid.uuid4().hex with event_dispatcher('srcservice', rabbit_config) as dispatch: dispatch(ExampleEvent(data)) # `handle` will have been called twice with the same the `data`, because # rabbit will have redelivered the un-ack'd message from the first call def event_handled_twice(): assert handle_called.call_args_list == [call(data), call(data)] assert_stops_raising(event_handled_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert queue_consumer_conn not in [conn['name'] for conn in connections]
def test_send_rpc_multi_message_reply_ignores_all_but_last(get_connection): queue_declared = Event() def response_greenthread(): with get_connection() as conn: with conn.channel() as chan: queue = nova.get_topic_queue('test_rpc', 'test', channel=chan) queue.declare() queue_declared.send(True) body, msg = ifirst( queue_iterator(queue, no_ack=True, timeout=2)) msgid, _, _, args = nova.parse_message(body) exchange = nova.get_reply_exchange(msgid) producer = Producer(chan, exchange=exchange, routing_key=msgid) for _ in range(3): msg = dict(result='should ignore this message', failure=None, ending=False) producer.publish(msg) eventlet.sleep(0.1) msg = dict(result=args, failure=None, ending=False) producer.publish(msg) msg = dict(result=None, failure=None, ending=True) producer.publish(msg) g = eventlet.spawn_n(response_greenthread) eventlet.sleep() with get_connection() as conn: ctx = context.get_admin_context() queue_declared.wait() resp = nova.send_rpc(conn, context=ctx, exchange='test_rpc', topic='test', method='test_method', args={ 'spam': 'shrub', }, timeout=3) assert resp == { 'spam': 'shrub', } eventlet.sleep() def check_greenthread_dead(): assert not g assert_stops_raising(check_greenthread_dead)
def test_service_disconnect_with_active_rpc_worker_via_service_proxy( logger, container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active rpc worker (i.e. response required). Make the rpc call from a nameko service. We expect the service to see the duplicate response and discard it. """ # ExampleService is the target; ProxyService has the rpc_proxy; proxy_container = container_factory(ProxyService, rabbit_config) service_container = container_factory(ExampleService, rabbit_config) service_container.start() # get exampleservice's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 service_consumer_conn = connections[0]['name'] proxy_container.start() # there should now be two connections: # 1. the queue consumer from proxyservice # 2. the queue consumer from exampleservice connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 # disconnect exampleservice's queue consumer while it's running the worker eventlet.spawn(disconnect_on_event, rabbit_manager, service_consumer_conn) # we should receive the response from the first call # the service rpc_proxy will receive and discard the response from the # second call arg = uuid.uuid4().hex with entrypoint_hook(proxy_container, 'entrypoint') as entrypoint: # we should receive a response after reconnection assert entrypoint(arg) == arg def duplicate_response_received(): correlation_warning = call("Unknown correlation id: %s", ANY) assert correlation_warning in logger.debug.call_args_list assert_stops_raising(duplicate_response_received) # `method` will have been called twice with the same the `arg`, because # rabbit will have redelivered the un-ack'd message from the first call def method_called_twice(): assert method_called.call_args_list == [call(arg), call(arg)] assert_stops_raising(method_called_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert service_consumer_conn not in [conn['name'] for conn in connections]
def test_send_rpc_multi_message_reply_ignores_all_but_last(get_connection): queue_declared = Event() def response_greenthread(): with get_connection() as conn: with conn.channel() as chan: queue = nova.get_topic_queue( 'test_rpc', 'test', channel=chan) queue.declare() queue_declared.send(True) body, msg = ifirst( queue_iterator(queue, no_ack=True, timeout=2)) msgid, _, _, args = nova.parse_message(body) exchange = nova.get_reply_exchange(msgid) producer = Producer(chan, exchange=exchange, routing_key=msgid) for _ in range(3): msg = dict( result='should ignore this message', failure=None, ending=False) producer.publish(msg) eventlet.sleep(0.1) msg = dict(result=args, failure=None, ending=False) producer.publish(msg) msg = dict(result=None, failure=None, ending=True) producer.publish(msg) g = eventlet.spawn_n(response_greenthread) eventlet.sleep() with get_connection() as conn: ctx = context.get_admin_context() queue_declared.wait() resp = nova.send_rpc( conn, context=ctx, exchange='test_rpc', topic='test', method='test_method', args={'spam': 'shrub', }, timeout=3) assert resp == {'spam': 'shrub', } eventlet.sleep() def check_greenthread_dead(): assert not g assert_stops_raising(check_greenthread_dead)
def test_multiple_runners_coexist(runner_factory, rabbit_config, rabbit_manager): runner1 = runner_factory(rabbit_config, Service) runner1.start() runner2 = runner_factory(rabbit_config, Service) runner2.start() vhost = rabbit_config['vhost'] # verify there are two event queues with a single consumer each def check_consumers(): evt_queues = [ queue for queue in rabbit_manager.get_queues(vhost) if queue['name'].startswith('evt-srcservice-testevent') ] assert len(evt_queues) == 2 for queue in evt_queues: assert queue['consumers'] == 1 # rabbit's management API seems to lag assert_stops_raising(check_consumers) # test events (both services will receive if in "broadcast" mode) event_data = "msg" dispatch = event_dispatcher(rabbit_config) dispatch('srcservice', "testevent", event_data) with eventlet.Timeout(1): while len(received) < 2: eventlet.sleep() assert received == [event_data, event_data] # verify there are two consumers on the rpc queue rpc_queue = rabbit_manager.get_queue(vhost, 'rpc-service') assert rpc_queue['consumers'] == 2 # test rpc (only one service will respond) del received[:] arg = "msg" with ServiceRpcProxy('service', rabbit_config) as proxy: proxy.handle(arg) with eventlet.Timeout(1): while len(received) == 0: eventlet.sleep() assert received == [arg]
def test_multiple_runners_coexist( runner_factory, rabbit_config, rabbit_manager ): runner1 = runner_factory(rabbit_config, Service) runner1.start() runner2 = runner_factory(rabbit_config, Service) runner2.start() vhost = rabbit_config['vhost'] # verify there are two event queues with a single consumer each def check_consumers(): evt_queues = [queue for queue in rabbit_manager.get_queues(vhost) if queue['name'].startswith('evt-srcservice-testevent')] assert len(evt_queues) == 2 for queue in evt_queues: assert queue['consumers'] == 1 # rabbit's management API seems to lag assert_stops_raising(check_consumers) # test events (both services will receive if in "broadcast" mode) event_data = "msg" dispatch = event_dispatcher(rabbit_config) dispatch('srcservice', "testevent", event_data) with eventlet.Timeout(1): while len(received) < 2: eventlet.sleep() assert received == [event_data, event_data] # verify there are two consumers on the rpc queue rpc_queue = rabbit_manager.get_queue(vhost, 'rpc-service') assert rpc_queue['consumers'] == 2 # test rpc (only one service will respond) del received[:] arg = "msg" with ServiceRpcProxy('service', rabbit_config) as proxy: proxy.handle(arg) with eventlet.Timeout(1): while len(received) == 0: eventlet.sleep() assert received == [arg]
def test_multiple_runners_coexist(runner_factory, rabbit_config, rabbit_manager, service_cls, tracker): runner1 = runner_factory(rabbit_config, service_cls) runner1.start() runner2 = runner_factory(rabbit_config, service_cls) runner2.start() vhost = rabbit_config['vhost'] # verify there are two event queues with a single consumer each def check_consumers(): evt_queues = [ queue for queue in rabbit_manager.get_queues(vhost) if queue['name'].startswith('evt-srcservice-testevent') ] assert len(evt_queues) == 2 for queue in evt_queues: assert queue['consumers'] == 1 # rabbit's management API seems to lag assert_stops_raising(check_consumers) # test events (both services will receive if in "broadcast" mode) event_data = "event" dispatch = event_dispatcher(rabbit_config) container1 = list(runner1.containers)[0] container2 = list(runner2.containers)[0] with entrypoint_waiter(container1, "handle"): with entrypoint_waiter(container2, "handle"): dispatch('srcservice', "testevent", event_data) assert tracker.call_args_list == [call(event_data), call(event_data)] # verify there are two consumers on the rpc queue rpc_queue = rabbit_manager.get_queue(vhost, 'rpc-service') assert rpc_queue['consumers'] == 2 # test rpc (only one service will respond) arg = "arg" with ServiceRpcProxy('service', rabbit_config) as proxy: proxy.handle(arg) assert tracker.call_args_list == [ call(event_data), call(event_data), call(arg) ]
def test_send_rpc_errors(get_connection): queue_declared = Event() def response_greenthread(): with get_connection() as conn: with conn.channel() as chan: queue = nova.get_topic_queue('test_rpc', 'test', channel=chan) queue.declare() queue_declared.send(True) body, msg = ifirst( queue_iterator(queue, no_ack=True, timeout=2)) msgid, _, _, _ = nova.parse_message(body) exchange = nova.get_reply_exchange(msgid) producer = Producer(chan, exchange=exchange, routing_key=msgid) exc = Exception('error') failure = (type(exc).__name__, str(exc)) msg = {'result': None, 'failure': failure, 'ending': False} producer.publish(msg) msg = {'result': None, 'failure': None, 'ending': True} producer.publish(msg) g = eventlet.spawn_n(response_greenthread) eventlet.sleep(0) with get_connection() as conn: ctx = context.get_admin_context() with pytest.raises(RemoteError): queue_declared.wait() nova.send_rpc(conn, context=ctx, exchange='test_rpc', topic='test', method='test_method', args={ 'foo': 'bar', }, timeout=3) def check_greenthread_dead(): assert not g assert_stops_raising(check_greenthread_dead)
def test_multiple_runners_coexist( runner_factory, rabbit_config, rabbit_manager, service_cls, tracker ): runner1 = runner_factory(rabbit_config, service_cls) runner1.start() runner2 = runner_factory(rabbit_config, service_cls) runner2.start() vhost = rabbit_config['vhost'] # verify there are two event queues with a single consumer each def check_consumers(): evt_queues = [queue for queue in rabbit_manager.get_queues(vhost) if queue['name'].startswith('evt-srcservice-testevent')] assert len(evt_queues) == 2 for queue in evt_queues: assert queue['consumers'] == 1 # rabbit's management API seems to lag assert_stops_raising(check_consumers) # test events (both services will receive if in "broadcast" mode) event_data = "event" dispatch = event_dispatcher(rabbit_config) container1 = list(runner1.containers)[0] container2 = list(runner2.containers)[0] with entrypoint_waiter(container1, "handle"): with entrypoint_waiter(container2, "handle"): dispatch('srcservice', "testevent", event_data) assert tracker.call_args_list == [call(event_data), call(event_data)] # verify there are two consumers on the rpc queue rpc_queue = rabbit_manager.get_queue(vhost, 'rpc-service') assert rpc_queue['consumers'] == 2 # test rpc (only one service will respond) arg = "arg" with ServiceRpcProxy('service', rabbit_config) as proxy: proxy.handle(arg) assert tracker.call_args_list == [ call(event_data), call(event_data), call(arg) ]
def test_send_rpc_errors(get_connection): queue_declared = Event() def response_greenthread(): with get_connection() as conn: with conn.channel() as chan: queue = nova.get_topic_queue( 'test_rpc', 'test', channel=chan) queue.declare() queue_declared.send(True) body, msg = ifirst( queue_iterator(queue, no_ack=True, timeout=2)) msgid, _, _, _ = nova.parse_message(body) exchange = nova.get_reply_exchange(msgid) producer = Producer(chan, exchange=exchange, routing_key=msgid) exc = Exception('error') failure = (type(exc).__name__, str(exc)) msg = {'result': None, 'failure': failure, 'ending': False} producer.publish(msg) msg = {'result': None, 'failure': None, 'ending': True} producer.publish(msg) g = eventlet.spawn_n(response_greenthread) eventlet.sleep(0) with get_connection() as conn: ctx = context.get_admin_context() with pytest.raises(RemoteError): queue_declared.wait() nova.send_rpc( conn, context=ctx, exchange='test_rpc', topic='test', method='test_method', args={'foo': 'bar', }, timeout=3) def check_greenthread_dead(): assert not g assert_stops_raising(check_greenthread_dead)
def test_send_rpc(get_connection): queue_declared = Event() def response_greenthread(): with get_connection() as conn: with conn.channel() as chan: queue = nova.get_topic_queue('test_rpc', 'test', channel=chan) queue.declare() queue_declared.send(True) msg = ifirst(queue_iterator(queue, no_ack=True, timeout=2)) msgid, _, _, args = nova.parse_message(msg.payload) exchange = nova.get_reply_exchange(msgid) producer = Producer(chan, exchange=exchange, routing_key=msgid) msg = {'result': args, 'failure': None, 'ending': False} producer.publish(msg) msg = {'result': None, 'failure': None, 'ending': True} producer.publish(msg) g = eventlet.spawn_n(response_greenthread) eventlet.sleep(0) with get_connection() as conn: ctx = context.get_admin_context() queue_declared.wait() resp = nova.send_rpc(conn, context=ctx, exchange='test_rpc', topic='test', method='test_method', args={ 'foo': 'bar', }, timeout=3) assert resp == { 'foo': 'bar', } def check_greenthread_dead(): assert not g assert_stops_raising(check_greenthread_dead)
def test_service_disconnect_with_active_rpc_worker(container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active rpc worker (i.e. response required). """ container = container_factory(ExampleService, rabbit_config) container.start() # get the service's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 queue_consumer_conn = connections[0]['name'] # create a standalone RPC proxy towards the target service rpc_proxy = RpcProxy('exampleservice', rabbit_config) proxy = rpc_proxy.start() # there should now be two connections: # 1. the queue consumer from the target service # 2. the queue consumer in the standalone rpc proxy connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 # disconnect the service's queue consumer while it's running a worker eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn) # we should receive the response from the first call # the standalone RPC proxy will stop listening as soon as it receives # a reply, so the duplicate response is discarded arg = uuid.uuid4().hex assert proxy.method(arg) == arg # `method` will have been called twice with the same the `arg`, because # rabbit will have redelivered the un-ack'd message from the first call def method_called_twice(): assert method_called.call_args_list == [call(arg), call(arg)] assert_stops_raising(method_called_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert queue_consumer_conn not in [conn['name'] for conn in connections] rpc_proxy.stop()
def test_service_disconnect_with_active_rpc_worker( container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active rpc worker (i.e. response required). """ container = container_factory(ExampleService, rabbit_config) container.start() # get the service's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 queue_consumer_conn = connections[0]['name'] # create a standalone RPC proxy towards the target service rpc_proxy = ServiceRpcProxy('exampleservice', rabbit_config) proxy = rpc_proxy.start() # there should now be two connections: # 1. the queue consumer from the target service # 2. the queue consumer in the standalone rpc proxy connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 # disconnect the service's queue consumer while it's running a worker eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn) # we should receive the response from the first call # the standalone RPC proxy will stop listening as soon as it receives # a reply, so the duplicate response is discarded arg = uuid.uuid4().hex assert proxy.method(arg) == arg # `method` will have been called twice with the same the `arg`, because # rabbit will have redelivered the un-ack'd message from the first call def method_called_twice(): assert method_called.call_args_list == [call(arg), call(arg)] assert_stops_raising(method_called_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert queue_consumer_conn not in [conn['name'] for conn in connections] rpc_proxy.stop()
def test_send_rpc(get_connection): queue_declared = Event() def response_greenthread(): with get_connection() as conn: with conn.channel() as chan: queue = nova.get_topic_queue( 'test_rpc', 'test', channel=chan) queue.declare() queue_declared.send(True) msg = ifirst(queue_iterator(queue, no_ack=True, timeout=2)) msgid, _, _, args = nova.parse_message(msg.payload) exchange = nova.get_reply_exchange(msgid) producer = Producer(chan, exchange=exchange, routing_key=msgid) msg = {'result': args, 'failure': None, 'ending': False} producer.publish(msg) msg = {'result': None, 'failure': None, 'ending': True} producer.publish(msg) g = eventlet.spawn_n(response_greenthread) eventlet.sleep(0) with get_connection() as conn: ctx = context.get_admin_context() queue_declared.wait() resp = nova.send_rpc( conn, context=ctx, exchange='test_rpc', topic='test', method='test_method', args={'foo': 'bar', }, timeout=3) assert resp == {'foo': 'bar', } def check_greenthread_dead(): assert not g assert_stops_raising(check_greenthread_dead)
def test_nameko_service_rpc_methods_are_called_by_any_wampy_client( runner_factory, router, config_path): wamp_client = Client(router=router) config = { WAMP_CONFIG_KEY: { 'config_path': config_path, } } runner = runner_factory(config, WampServiceA) runner.start() assert WampServiceA.messages == [] container = get_container(runner, WampServiceA) with wamp_client as client: wait_for_session(client) wait_for_registrations(container, number_of_registrations=1) wait_for_subscriptions(container, number_of_subscriptions=2) result = client.rpc.spam_call(cheese="cheddar", eggs="ducks") assert result == "spam" def waiting_for_the_message(): assert len(WampServiceA.messages) == 1 assert WampServiceA.messages == [((), { u'cheese': u'cheddar', u'eggs': u'ducks' })] assert_stops_raising(waiting_for_the_message) container.stop() WampServiceA.messages = []