async def test_discovery_with_a_static_seed(): """ When we create a discoverer with a static seed address we should query that address for gossip and then select the best node from the result. On a subsequent call, we should use the previously discovered nodes to find gossip. """ seed_ip = "10.10.10.10" first_node_ip = "172.31.0.1" second_node_ip = "192.168.168.192" first_gossip = data.make_gossip(first_node_ip) second_gossip = data.make_gossip(second_node_ip) discoverer = get_discoverer(None, None, seed_ip, 2113) with aioresponses() as mock: mock.get(f"http://{seed_ip}:2113/gossip", status=200, payload=first_gossip) mock.get(f"http://{first_node_ip}:2113/gossip", status=200, payload=second_gossip) assert await discoverer.next_node() == NodeService( first_node_ip, 1113, None) assert await discoverer.next_node() == NodeService( second_node_ip, 1113, None) assert discoverer.retry_policy.retries_per_node == 3
async def test_repeated_discovery_failure_for_static_seed(): """ When gossip fetch fails `maximum_retry_count` times, we should fail with a DiscoverFailed error. """ class always_fail(DiscoveryRetryPolicy): def __init__(self): super().__init__() def should_retry(self, _): return False async def wait(self, seed): pass seed = NodeService("1.2.3.4", 2113, None) retry = always_fail() gossip = data.make_gossip("2.3.4.5") with aioresponses() as mock: successful_discoverer = ClusterDiscovery(StaticSeedFinder([seed]), retry, None) mock.get("http://1.2.3.4:2113/gossip", status=500) mock.get("http://1.2.3.4:2113/gossip", payload=gossip) with pytest.raises(DiscoveryFailed): assert await successful_discoverer.next_node() == NodeService( "2.3.4.5", 1113, None) stats = retry.stats[seed] assert stats.attempts == 1 assert stats.successes == 0 assert stats.failures == 1 assert stats.consecutive_failures == 1
async def test_discovery_failure_for_static_seed(): """ When gossip fetch fails for a static seed, we should call the retry thing """ class always_succeed(DiscoveryRetryPolicy): def __init__(self): super().__init__() self.stats = Stats() def should_retry(self, _): return True async def wait(self, seed): pass seed = NodeService("1.2.3.4", 2113, None) gossip = data.make_gossip("2.3.4.5") retry = always_succeed() with aioresponses() as mock: successful_discoverer = ClusterDiscovery(StaticSeedFinder([seed]), retry, None) mock.get("http://1.2.3.4:2113/gossip", status=500) mock.get("http://1.2.3.4:2113/gossip", payload=gossip) assert await successful_discoverer.next_node() == NodeService( "2.3.4.5", 1113, None) stats = retry.stats[seed] assert stats.attempts == 2 assert stats.successes == 1 assert stats.failures == 1 assert stats.consecutive_failures == 0
async def test_when_restarting_a_running_connector(event_loop): queue = TeeQueue() addr = NodeService("localhost", 8338, None) dispatcher = SpyDispatcher() connector = Connector( SingleNodeDiscovery(addr, DiscoveryRetryPolicy()), dispatcher, loop=event_loop, ctrl_queue=queue, ) async with EchoServer(addr, event_loop): await connector.start() [connect, connected] = await queue.next_event(count=2) assert connected.command == ConnectorCommand.HandleConnectionOpened await connector_event(connector.connected) await connector.reconnect(connector.target_node) [connect, closed, b, c, d, connected] = await queue.next_event(count=6) assert closed.command == ConnectorCommand.HandleConnectionClosed assert connect.command == ConnectorCommand.Connect assert connected.command == ConnectorCommand.HandleConnectionOpened await connector.stop()
async def test_when_the_connection_fails_with_an_error(event_loop): queue = TeeQueue() addr = NodeService("localhost", 8338, None) dispatcher = SpyDispatcher() connector = Connector( SingleNodeDiscovery(addr, DiscoveryRetryPolicy()), dispatcher, loop=event_loop, ctrl_queue=queue, ) async with EchoServer(addr, event_loop): await connector.start() [connect, connected] = await queue.next_event(count=2) exn = ValueError() connector.connection_lost(exn) [connection_failed] = await queue.next_event(count=1) assert connection_failed.command == ConnectorCommand.HandleConnectionFailed assert connection_failed.data is exn await connector.stop()
async def test_when_three_heartbeats_fail_in_a_row(event_loop): """ We're going to set up a separate heartbeat loop to send heartbeat requests to the server. If three of those heartbeats timeout in a row, we'll put a reconnection request on the queue. """ queue = TeeQueue() addr = NodeService("localhost", 8338, None) dispatcher = SpyDispatcher() connector = Connector(SingleNodeDiscovery(addr), dispatcher, loop=event_loop, ctrl_queue=queue) async with EchoServer(addr, event_loop): await connector.start() [connect, connected] = await queue.next_event(count=2) assert connect.command == ConnectorCommand.Connect assert connected.command == ConnectorCommand.HandleConnectionOpened connector.heartbeat_failed() connector.heartbeat_failed() connector.heartbeat_failed() [hb1, hb2, hb3, connection_closed, reconnect] = await queue.next_event(count=5) assert connection_closed.command == ConnectorCommand.HandleConnectionClosed assert reconnect.command == ConnectorCommand.Connect await connector.stop()
async def test_when_connecting_to_a_server(event_loop): """ When we connect to a server, the protocol should begin sending the pending messages held by the dispatcher. When it receives an InboundMessage from the MessageReader, it should call the "dispatch" method of the Dispatcher. """ addr = NodeService("localhost", 8338, None) async with EchoServer(addr, event_loop): dispatcher = SpyDispatcher() connector = Connector(SingleNodeDiscovery(addr), dispatcher, loop=event_loop) ping = Ping() await dispatcher.start_conversation(ping) await connector.start() await connector_event(connector.connected) roundtripped_message = await dispatcher.received.get() assert roundtripped_message.conversation_id == ping.conversation_id await connector.stop()
async def test_fetch_gossip(): node = NodeService(address="10.10.10.10", port=2113, secure_port=None) with aioresponses() as mock: mock.get("http://10.10.10.10:2113/gossip", status=200, payload=data.GOSSIP) async with aiohttp.ClientSession() as session: gossip = await fetch_new_gossip(session, node) assert len(gossip) == 3
async def test_aiohttp_failure(): node = NodeService(address="10.10.10.10", port=2113, secure_port=None) with aioresponses() as mock: mock.get("http://10.10.10.10:2113/gossip", status=502) async with aiohttp.ClientSession() as session: gossip = await fetch_new_gossip(session, node) assert not gossip
async def test_discovery_with_a_single_node(): """ When we create a discoverer with a single address and no discovery information, we should receive an async generator that returns the same node over and over. """ discoverer = get_discoverer("localhost", 1113, None, None) for i in range(0, 5): assert await discoverer.discover() == NodeService("localhost", 1113, None)
async def test_prefer_master(): """ If we ask the discoverer to prefer_master it should return a master node before returning a replica. """ discoverer = get_discoverer(None, None, "10.0.0.1", 2113, prefer_master) gossip = data.make_gossip("10.0.0.1", "10.0.0.2") with aioresponses() as mock: mock.get("http://10.0.0.1:2113/gossip", payload=gossip) assert await discoverer.next_node() == NodeService("10.0.0.1", 1113, None)
async def test_single_node_mark_failed(): """ The SingleNodeDiscovery should raise DiscoveryFailed if we ask for a node after calling mark_failed. """ node = NodeService("2.3.4.5", 1234, None) discoverer = SingleNodeDiscovery(node) assert await discoverer.discover() == node discoverer.mark_failed(node) with pytest.raises(DiscoveryFailed): await discoverer.discover()
async def test_cluster_discovery_mark_failed(): """ ClusterDiscovery should just pass the mark_failed call to the seed source. """ class spy_seed_finder(List): def mark_failed(self, node): self.append(node) node = NodeService("2.3.4.5", 1234, None) finder = spy_seed_finder() discoverer = ClusterDiscovery(finder, None, None, None) discoverer.mark_failed(node) assert finder == [node]
async def test_when_restarting_a_stopped_connector(event_loop): queue = TeeQueue() addr = NodeService("localhost", 8338, None) dispatcher = SpyDispatcher() connector = Connector(SingleNodeDiscovery(addr), dispatcher, loop=event_loop, ctrl_queue=queue) async with EchoServer(addr, event_loop): await connector.reconnect() [connect, connected] = await queue.next_event(count=2) assert connect.command == ConnectorCommand.Connect assert connected.command == ConnectorCommand.HandleConnectionOpened await connector_event(connector.connected) await connector.stop()
async def test_when_a_server_disconnects(event_loop): """ Usually, when eventstore goes away, we'll get an EOF on the transport. If that happens, we should raise a disconnected event. We should also place a reconnection message on the control queue. """ addr = NodeService("localhost", 8338, None) queue = TeeQueue() dispatcher = SpyDispatcher() connector = Connector( SingleNodeDiscovery(addr, DiscoveryRetryPolicy()), dispatcher, loop=event_loop, ctrl_queue=queue, ) raised_disconnected_event = asyncio.Future(loop=event_loop) def on_disconnected(): raised_disconnected_event.set_result(True) connector.disconnected.append(on_disconnected) async with EchoServer(addr, event_loop) as server: await connector.start() connect = await queue.next_event() assert connect.command == ConnectorCommand.Connect connected = await queue.next_event() assert connected.command == ConnectorCommand.HandleConnectionOpened server.stop() disconnect = await queue.next_event() assert disconnect.command == ConnectorCommand.HandleConnectionFailed reconnect = await queue.next_event() assert reconnect.command == ConnectorCommand.Connect assert raised_disconnected_event.result() is True await connector.stop()
async def test_when_a_heartbeat_succeeds(event_loop): """ If one of our heartbeats succeeds, we should reset our counter. Ergo, if we have two failures, followed by a success, followed by two failures, we should not reset the connection. """ queue = TeeQueue() addr = NodeService("localhost", 8338, None) dispatcher = SpyDispatcher() connector = Connector( SingleNodeDiscovery(addr, DiscoveryRetryPolicy()), dispatcher, loop=event_loop, ctrl_queue=queue, ) async with EchoServer(addr, event_loop): await connector.start() [connect, connected] = await queue.next_event(count=2) assert connect.command == ConnectorCommand.Connect assert connected.command == ConnectorCommand.HandleConnectionOpened connector.heartbeat_failed() connector.heartbeat_failed() [hb1, hb2] = await queue.next_event(count=2) assert connector.heartbeat_failures == 2 connector.heartbeat_received("Foo") connector.heartbeat_failed() connector.heartbeat_failed() [success, hb3, hb4] = await queue.next_event(count=3) assert connector.heartbeat_failures == 2 assert success.command == ConnectorCommand.HandleHeartbeatSuccess assert hb3.command == ConnectorCommand.HandleHeartbeatFailed assert hb4.command == ConnectorCommand.HandleHeartbeatFailed await connector.stop()
async def test_when_discovery_fails_on_reconnection(event_loop): """ If we can't retry our current node any more, and we can't discover a new one then it's game over and we should raise the stopped event. """ class never_retry: def __init__(self): self.recorded = None def should_retry(self, _): return self.recorded is None def record_failure(self, node): self.recorded = node async def wait(self, node): ... wait_for_stopped = asyncio.Future() def on_stopped(exn): wait_for_stopped.set_result(exn) queue = TeeQueue() addr = NodeService("localhost", 8338, None) policy = never_retry() dispatcher = SpyDispatcher() connector = Connector(SingleNodeDiscovery(addr, policy), dispatcher, loop=event_loop, ctrl_queue=queue) connector.stopped.append(on_stopped) await connector.start() [connect, connection_failed] = await queue.next_event(count=2) [failed] = await asyncio.wait_for(queue.next_event(count=1), 2) assert failed.command == ConnectorCommand.HandleConnectorFailed assert policy.recorded == addr assert isinstance(await wait_for_stopped, DiscoveryFailed)
read_gossip, select, prefer_master, prefer_replica, KEEP_RETRYING, ) from . import data GOOD_NODE = DiscoveredNode( state=NodeState.Master, is_alive=True, internal_tcp=None, internal_http=None, external_http=None, external_tcp=NodeService("10.128.10.10", 1113, None), ) def test_selector_with_one_node(): gossip = [GOOD_NODE] selected = select(gossip) assert selected == GOOD_NODE def test_selector_with_empty_gossip(): gossip = []