async def _probe_node_indirect_via(self, target_node: state.Node, indirect_node: state.Node):
LOG.debug("Probing node: %s indirectly via %s", target_node.name, indirect_node.name)
# get a sequence number for the ping
next_seq = self._probe_seq.increment()
# start waiting for probe result
waiter = self._wait_for_probe(next_seq)
# send ping request message
ping_req = messages.PingRequestMessage(next_seq,
target=target_node.name,
target_addr=messages.InternetAddress(target_node.host,
target_node.port),
sender=self.local_node_name,
sender_addr=messages.InternetAddress(self.local_node_address,
self.local_node_port))
LOG.debug("Sending PING-REQ (seq=%d) to %s", ping_req.seq, indirect_node.name)
await self._send_udp_message(indirect_node.host, indirect_node.port, ping_req)
# wait for probe result or timeout
try:
result = await waiter
except asyncio.TimeoutError:
LOG.debug("Timeout waiting for indirect ACK %d", next_seq)
await self._handle_probe_timeout(target_node)
else:
await self._handle_probe_result(target_node, result)
async def _handle_ping_request_message(self, msg, client_addr):
LOG.trace("Handling PING-REQ (%d): target=%s", msg.seq, msg.node)
# get a sequence number for the ping
next_seq = self._probe_seq.increment()
# start waiting for probe result
waiter = self._wait_for_probe(next_seq)
# create PingMessage
ping = messages.PingMessage(next_seq,
target=msg.node,
sender=self.local_node_name,
sender_addr=messages.InternetAddress(self.local_node_address,
self.local_node_port))
LOG.debug("Sending PING (%d) to %s in response to PING-REQ (%d)", next_seq, msg.node_addr, msg.seq)
await self._send_udp_message(msg.node_addr.address, msg.node_addr.port, ping)
# wait for probe result or timeout
try:
result = await waiter
except asyncio.TimeoutError:
LOG.debug("Timeout waiting for ACK %d", next_seq)
# TODO: send nack
# await self._forward_indirect_probe_timeout(msg)
else:
await self._forward_indirect_probe_result(msg)
async def test_suspect_node_refute(self):
await self.nodes.on_node_suspect('local-node', 1)
local_node = self.nodes.local_node
self.assertEqual(local_node.incarnation, 2)
self.assertEqual(local_node.status, state.NODE_STATUS_ALIVE)
# suspecting the local node should be refuted (sand Alive)
expected_message = messages.AliveMessage('local-node', messages.InternetAddress('127.0.0.1', 7800), 2)
self.queue.push.assert_called_with('local-node', messages.MessageSerializer.encode(expected_message))
def test_encode_list(self):
state = [
messages.RemoteNodeState(
'test', messages.InternetAddress('127.0.0.0', 12345), 1, 1,
'alive', None)
]
orig = messages.SyncMessage(remote_state=state)
buf = messages.MessageSerializer.encode(orig)
new = messages.MessageSerializer.decode(buf)
self.assertEqual(orig, new)
async def test_alive_node(self):
await self.nodes.on_node_alive('node-1', 2, '127.0.0.1', 7801)
expected_node = self.nodes.get('node-1')
self.assertIsNotNone(expected_node)
self.assertEqual(expected_node.incarnation, 2)
self.assertEqual(expected_node.status, state.NODE_STATUS_ALIVE)
expected_message = messages.AliveMessage('node-1', messages.InternetAddress('127.0.0.1', 7801), 2)
self.queue.push.assert_called_with('node-1', messages.MessageSerializer.encode(expected_message))
async def _send_local_state(self, stream_writer):
# get local state
local_state = []
for node in self._nodes:
local_state.append(messages.RemoteNodeState(node=node.name,
addr=messages.InternetAddress(node.host, node.port),
version=node.version,
incarnation=node.incarnation,
status=node.status,
metadata=node.metadata))
LOG.trace("Sending local state %s", local_state)
# send message
await self._send_tcp_message(messages.SyncMessage(remote_state=local_state), stream_writer)
async def _probe_node(self, target_node: state.Node):
LOG.debug("Probing node: %s", target_node)
# get a sequence number for the ping
next_seq = self._probe_seq.increment()
# start waiting for probe result
waiter = self._wait_for_probe(next_seq)
# send ping message
ping = messages.PingMessage(next_seq,
target=target_node.name,
sender=self.local_node_name,
sender_addr=messages.InternetAddress(self.local_node_address,
self.local_node_port))
LOG.debug("Sending PING (seq=%d) to %s", ping.seq, target_node.name)
await self._send_udp_message(target_node.host, target_node.port, ping)
# wait for probe result or timeout
result = False
try:
result = await waiter
except asyncio.TimeoutError:
await self._handle_probe_timeout(target_node)
else:
await self._handle_probe_result(target_node, result)
# if probe was successful no need to send indirect probe
if result:
return
# if probe failed, send indirect probe
try:
result = await self._probe_node_indirect(target_node, self.config.probe_indirect_nodes)
except Exception:
LOG.exception("Error running indirect probe")
def _broadcast_alive(self, node):
LOG.debug("Broadcasting ALIVE message for node: %s (incarnation=%d)", node.name, node.incarnation)
alive = messages.AliveMessage(node=node.name,
addr=messages.InternetAddress(node.host, node.port),
incarnation=node.incarnation)
self._send_broadcast(node, alive)
def test_encode_complex(self):
orig = messages.PingRequestMessage(
1, "node", messages.InternetAddress('host', 123), "sender")
buf = messages.MessageSerializer.encode(orig)
self.assertEqual(orig, messages.MessageSerializer.decode(buf))