async def test_discovery_with_a_static_seed():
"""
When we create a discoverer with a static seed address we
should query that address for gossip and then select the
best node from the result.
On a subsequent call, we should use the previously discovered
nodes to find gossip.
"""
seed_ip = "10.10.10.10"
first_node_ip = "172.31.0.1"
second_node_ip = "192.168.168.192"
first_gossip = data.make_gossip(first_node_ip)
second_gossip = data.make_gossip(second_node_ip)
discoverer = get_discoverer(None, None, seed_ip, 2113)
with aioresponses() as mock:
mock.get(f"http://{seed_ip}:2113/gossip",
status=200,
payload=first_gossip)
mock.get(f"http://{first_node_ip}:2113/gossip",
status=200,
payload=second_gossip)
assert await discoverer.next_node() == NodeService(
first_node_ip, 1113, None)
assert await discoverer.next_node() == NodeService(
second_node_ip, 1113, None)
assert discoverer.retry_policy.retries_per_node == 3
async def test_repeated_discovery_failure_for_static_seed():
"""
When gossip fetch fails `maximum_retry_count` times, we should fail with a
DiscoverFailed error.
"""
class always_fail(DiscoveryRetryPolicy):
def __init__(self):
super().__init__()
def should_retry(self, _):
return False
async def wait(self, seed):
pass
seed = NodeService("1.2.3.4", 2113, None)
retry = always_fail()
gossip = data.make_gossip("2.3.4.5")
with aioresponses() as mock:
successful_discoverer = ClusterDiscovery(StaticSeedFinder([seed]),
retry, None)
mock.get("http://1.2.3.4:2113/gossip", status=500)
mock.get("http://1.2.3.4:2113/gossip", payload=gossip)
with pytest.raises(DiscoveryFailed):
assert await successful_discoverer.next_node() == NodeService(
"2.3.4.5", 1113, None)
stats = retry.stats[seed]
assert stats.attempts == 1
assert stats.successes == 0
assert stats.failures == 1
assert stats.consecutive_failures == 1
async def test_discovery_failure_for_static_seed():
"""
When gossip fetch fails for a static seed, we should call the retry thing
"""
class always_succeed(DiscoveryRetryPolicy):
def __init__(self):
super().__init__()
self.stats = Stats()
def should_retry(self, _):
return True
async def wait(self, seed):
pass
seed = NodeService("1.2.3.4", 2113, None)
gossip = data.make_gossip("2.3.4.5")
retry = always_succeed()
with aioresponses() as mock:
successful_discoverer = ClusterDiscovery(StaticSeedFinder([seed]),
retry, None)
mock.get("http://1.2.3.4:2113/gossip", status=500)
mock.get("http://1.2.3.4:2113/gossip", payload=gossip)
assert await successful_discoverer.next_node() == NodeService(
"2.3.4.5", 1113, None)
stats = retry.stats[seed]
assert stats.attempts == 2
assert stats.successes == 1
assert stats.failures == 1
assert stats.consecutive_failures == 0
async def test_when_restarting_a_running_connector(event_loop):
queue = TeeQueue()
addr = NodeService("localhost", 8338, None)
dispatcher = SpyDispatcher()
connector = Connector(
SingleNodeDiscovery(addr, DiscoveryRetryPolicy()),
dispatcher,
loop=event_loop,
ctrl_queue=queue,
)
async with EchoServer(addr, event_loop):
await connector.start()
[connect, connected] = await queue.next_event(count=2)
assert connected.command == ConnectorCommand.HandleConnectionOpened
await connector_event(connector.connected)
await connector.reconnect(connector.target_node)
[connect, closed, b, c, d, connected] = await queue.next_event(count=6)
assert closed.command == ConnectorCommand.HandleConnectionClosed
assert connect.command == ConnectorCommand.Connect
assert connected.command == ConnectorCommand.HandleConnectionOpened
await connector.stop()
async def test_when_the_connection_fails_with_an_error(event_loop):
queue = TeeQueue()
addr = NodeService("localhost", 8338, None)
dispatcher = SpyDispatcher()
connector = Connector(
SingleNodeDiscovery(addr, DiscoveryRetryPolicy()),
dispatcher,
loop=event_loop,
ctrl_queue=queue,
)
async with EchoServer(addr, event_loop):
await connector.start()
[connect, connected] = await queue.next_event(count=2)
exn = ValueError()
connector.connection_lost(exn)
[connection_failed] = await queue.next_event(count=1)
assert connection_failed.command == ConnectorCommand.HandleConnectionFailed
assert connection_failed.data is exn
await connector.stop()
async def test_when_three_heartbeats_fail_in_a_row(event_loop):
"""
We're going to set up a separate heartbeat loop to send heartbeat requests
to the server. If three of those heartbeats timeout in a row, we'll put a
reconnection request on the queue.
"""
queue = TeeQueue()
addr = NodeService("localhost", 8338, None)
dispatcher = SpyDispatcher()
connector = Connector(SingleNodeDiscovery(addr),
dispatcher,
loop=event_loop,
ctrl_queue=queue)
async with EchoServer(addr, event_loop):
await connector.start()
[connect, connected] = await queue.next_event(count=2)
assert connect.command == ConnectorCommand.Connect
assert connected.command == ConnectorCommand.HandleConnectionOpened
connector.heartbeat_failed()
connector.heartbeat_failed()
connector.heartbeat_failed()
[hb1, hb2, hb3, connection_closed,
reconnect] = await queue.next_event(count=5)
assert connection_closed.command == ConnectorCommand.HandleConnectionClosed
assert reconnect.command == ConnectorCommand.Connect
await connector.stop()
async def test_when_connecting_to_a_server(event_loop):
"""
When we connect to a server, the protocol should begin sending the
pending messages held by the dispatcher.
When it receives an InboundMessage from the MessageReader, it should call
the "dispatch" method of the Dispatcher.
"""
addr = NodeService("localhost", 8338, None)
async with EchoServer(addr, event_loop):
dispatcher = SpyDispatcher()
connector = Connector(SingleNodeDiscovery(addr),
dispatcher,
loop=event_loop)
ping = Ping()
await dispatcher.start_conversation(ping)
await connector.start()
await connector_event(connector.connected)
roundtripped_message = await dispatcher.received.get()
assert roundtripped_message.conversation_id == ping.conversation_id
await connector.stop()
async def test_fetch_gossip():
node = NodeService(address="10.10.10.10", port=2113, secure_port=None)
with aioresponses() as mock:
mock.get("http://10.10.10.10:2113/gossip", status=200, payload=data.GOSSIP)
async with aiohttp.ClientSession() as session:
gossip = await fetch_new_gossip(session, node)
assert len(gossip) == 3
async def test_aiohttp_failure():
node = NodeService(address="10.10.10.10", port=2113, secure_port=None)
with aioresponses() as mock:
mock.get("http://10.10.10.10:2113/gossip", status=502)
async with aiohttp.ClientSession() as session:
gossip = await fetch_new_gossip(session, node)
assert not gossip
async def test_discovery_with_a_single_node():
"""
When we create a discoverer with a single address and no
discovery information, we should receive an async generator
that returns the same node over and over.
"""
discoverer = get_discoverer("localhost", 1113, None, None)
for i in range(0, 5):
assert await discoverer.discover() == NodeService("localhost", 1113, None)
async def test_prefer_master():
"""
If we ask the discoverer to prefer_master it should return a master node
before returning a replica.
"""
discoverer = get_discoverer(None, None, "10.0.0.1", 2113, prefer_master)
gossip = data.make_gossip("10.0.0.1", "10.0.0.2")
with aioresponses() as mock:
mock.get("http://10.0.0.1:2113/gossip", payload=gossip)
assert await discoverer.next_node() == NodeService("10.0.0.1", 1113, None)
async def test_single_node_mark_failed():
"""
The SingleNodeDiscovery should raise DiscoveryFailed if we ask for a node
after calling mark_failed.
"""
node = NodeService("2.3.4.5", 1234, None)
discoverer = SingleNodeDiscovery(node)
assert await discoverer.discover() == node
discoverer.mark_failed(node)
with pytest.raises(DiscoveryFailed):
await discoverer.discover()
async def test_cluster_discovery_mark_failed():
"""
ClusterDiscovery should just pass the mark_failed call to the seed source.
"""
class spy_seed_finder(List):
def mark_failed(self, node):
self.append(node)
node = NodeService("2.3.4.5", 1234, None)
finder = spy_seed_finder()
discoverer = ClusterDiscovery(finder, None, None, None)
discoverer.mark_failed(node)
assert finder == [node]
async def test_when_restarting_a_stopped_connector(event_loop):
queue = TeeQueue()
addr = NodeService("localhost", 8338, None)
dispatcher = SpyDispatcher()
connector = Connector(SingleNodeDiscovery(addr),
dispatcher,
loop=event_loop,
ctrl_queue=queue)
async with EchoServer(addr, event_loop):
await connector.reconnect()
[connect, connected] = await queue.next_event(count=2)
assert connect.command == ConnectorCommand.Connect
assert connected.command == ConnectorCommand.HandleConnectionOpened
await connector_event(connector.connected)
await connector.stop()
async def test_when_a_server_disconnects(event_loop):
"""
Usually, when eventstore goes away, we'll get an EOF on the transport.
If that happens, we should raise a disconnected event.
We should also place a reconnection message on the control queue.
"""
addr = NodeService("localhost", 8338, None)
queue = TeeQueue()
dispatcher = SpyDispatcher()
connector = Connector(
SingleNodeDiscovery(addr, DiscoveryRetryPolicy()),
dispatcher,
loop=event_loop,
ctrl_queue=queue,
)
raised_disconnected_event = asyncio.Future(loop=event_loop)
def on_disconnected():
raised_disconnected_event.set_result(True)
connector.disconnected.append(on_disconnected)
async with EchoServer(addr, event_loop) as server:
await connector.start()
connect = await queue.next_event()
assert connect.command == ConnectorCommand.Connect
connected = await queue.next_event()
assert connected.command == ConnectorCommand.HandleConnectionOpened
server.stop()
disconnect = await queue.next_event()
assert disconnect.command == ConnectorCommand.HandleConnectionFailed
reconnect = await queue.next_event()
assert reconnect.command == ConnectorCommand.Connect
assert raised_disconnected_event.result() is True
await connector.stop()
async def test_when_a_heartbeat_succeeds(event_loop):
"""
If one of our heartbeats succeeds, we should reset our counter.
Ergo, if we have two failures, followed by a success, followed
by two failures, we should not reset the connection.
"""
queue = TeeQueue()
addr = NodeService("localhost", 8338, None)
dispatcher = SpyDispatcher()
connector = Connector(
SingleNodeDiscovery(addr, DiscoveryRetryPolicy()),
dispatcher,
loop=event_loop,
ctrl_queue=queue,
)
async with EchoServer(addr, event_loop):
await connector.start()
[connect, connected] = await queue.next_event(count=2)
assert connect.command == ConnectorCommand.Connect
assert connected.command == ConnectorCommand.HandleConnectionOpened
connector.heartbeat_failed()
connector.heartbeat_failed()
[hb1, hb2] = await queue.next_event(count=2)
assert connector.heartbeat_failures == 2
connector.heartbeat_received("Foo")
connector.heartbeat_failed()
connector.heartbeat_failed()
[success, hb3, hb4] = await queue.next_event(count=3)
assert connector.heartbeat_failures == 2
assert success.command == ConnectorCommand.HandleHeartbeatSuccess
assert hb3.command == ConnectorCommand.HandleHeartbeatFailed
assert hb4.command == ConnectorCommand.HandleHeartbeatFailed
await connector.stop()
async def test_when_discovery_fails_on_reconnection(event_loop):
"""
If we can't retry our current node any more, and we can't discover a new one
then it's game over and we should raise the stopped event.
"""
class never_retry:
def __init__(self):
self.recorded = None
def should_retry(self, _):
return self.recorded is None
def record_failure(self, node):
self.recorded = node
async def wait(self, node):
...
wait_for_stopped = asyncio.Future()
def on_stopped(exn):
wait_for_stopped.set_result(exn)
queue = TeeQueue()
addr = NodeService("localhost", 8338, None)
policy = never_retry()
dispatcher = SpyDispatcher()
connector = Connector(SingleNodeDiscovery(addr, policy),
dispatcher,
loop=event_loop,
ctrl_queue=queue)
connector.stopped.append(on_stopped)
await connector.start()
[connect, connection_failed] = await queue.next_event(count=2)
[failed] = await asyncio.wait_for(queue.next_event(count=1), 2)
assert failed.command == ConnectorCommand.HandleConnectorFailed
assert policy.recorded == addr
assert isinstance(await wait_for_stopped, DiscoveryFailed)
read_gossip,
select,
prefer_master,
prefer_replica,
KEEP_RETRYING,
)
from . import data
GOOD_NODE = DiscoveredNode(
state=NodeState.Master,
is_alive=True,
internal_tcp=None,
internal_http=None,
external_http=None,
external_tcp=NodeService("10.128.10.10", 1113, None),
)
def test_selector_with_one_node():
gossip = [GOOD_NODE]
selected = select(gossip)
assert selected == GOOD_NODE
def test_selector_with_empty_gossip():
gossip = []
