async def _async_set_current_neighbours(
neighbours: t.List[Server] = None,
changed_routes: t.Dict[Server,
RouteContainer] = None) -> t.List[Server]:
"""Function checks and sets neighbours
Args:
neighbours: list of neighbours
changed_routes: reference to a dict which will be populated with new routes
Returns:
list of servers which are not neighbours anymore
"""
not_neighbours_anymore = []
if neighbours is None:
neighbours = Server.get_neighbours()
if neighbours:
resp = await asyncio.gather(
*[async_check_gates(server) for server in neighbours])
for route, server in zip(resp, neighbours):
if isinstance(route, RouteContainer):
server.set_route(route)
if changed_routes is not None:
changed_routes[server] = route
elif route is None:
not_neighbours_anymore.append(server)
rc = RouteContainer(None, None, None)
server.set_route(rc)
if changed_routes is not None:
changed_routes[server] = rc
return not_neighbours_anymore
def _remove_node_from_cluster(self, server_id):
changed_routes = {}
server = self.session.query(Server).get(server_id)
if server:
server.set_route(RouteContainer(None, None, None))
lost_routes = self.session.query(Route).filter_by(
proxy_server_id=server.id).options(
orm.lazyload(Route.destination), orm.lazyload(Route.gate),
orm.lazyload(Route.proxy_server)).count()
if lost_routes:
changed_routes = self._loop.run_until_complete(
self._async_refresh_route_table(
discover_new_neighbours=False,
check_current_neighbours=False))
changed_routes.update({server: RouteContainer(None, None, None)})
return changed_routes
async def async_check_gates(server,
timeout=5,
retries=1,
delay=1) -> t.Union[RouteContainer, Route]:
"""checks if a server is neighbour
Args:
server: server to discover as a new neighbour
timeout: timeout socket
retries: number of times to try to connect to the socket
delay: time in seconds between retries
Returns:
returns Route if no change. This means that gate in inventory is still up.
RouteContainer if there is an available gate to connect
None if no gate to connect
"""
# check current gate
if server.route and server.route.gate:
gate = server.route.gate
if await async_check_host(host=gate.dns or str(gate.ip),
port=gate.port,
retry=retries + 1,
delay=delay,
timeout=timeout):
return server.route
for gate in server.gates:
if server.route and gate != server.route.gate:
if (gate.ip and not gate.ip.is_loopback) or (
gate.dns and gate.dns != 'localhost'):
if await async_check_host(host=gate.dns or str(gate.ip),
port=gate.port,
retry=retries,
delay=1,
timeout=timeout):
return RouteContainer(None, gate, 0)
def _update_route_table_from_data(
self,
new_routes: t.Dict,
auth=None) -> t.Dict[Server, RouteContainer]:
changed_routes = {}
routes = new_routes.get('route_list', [])
routes.sort(key=lambda x: x.get('cost') or MAX_COST, reverse=True)
try:
likely_proxy_server = self.session.query(Server).get(
new_routes.get('server_id'))
if not likely_proxy_server:
self.logger.warning(
f"Server id still '{new_routes.get('server_id')}' not created."
)
return changed_routes
debug_new_routes = []
for new_route in routes:
target_server = self.session.query(Server).get(
new_route.get('destination_id'))
proxy_server = self.session.query(Server).get(
new_route.get('proxy_server_id'))
gate = self.session.query(Gate).get(new_route.get('gate_id'))
dest_name = getattr(target_server, 'name',
new_route.get('destination_id'))
proxy_name = getattr(proxy_server, 'name',
new_route.get('proxy_server_id'))
gate_str = str(gate) if gate else new_route.get('gate_id')
cost = new_route.get('cost')
if gate_str and proxy_name:
gate_str = gate_str + '*' + proxy_name
debug_new_routes.append(
f'{dest_name} -> {gate_str or proxy_name} / {cost}')
if target_server is None:
self.logger.warning(
f"Destination server unknown {new_route.get('destination_id')}"
)
continue
if target_server.id == self.server.id:
# check if server has detected me as a neighbour
if new_route.get('cost') == 0:
# check if I do not have it as a neighbour yet
if likely_proxy_server.route and likely_proxy_server.route.cost != 0:
# check if I have a gate to contact with it
route = check_gates(likely_proxy_server)
if isinstance(route, RouteContainer):
changed_routes[likely_proxy_server] = route
likely_proxy_server.set_route(route)
else:
# server may be created without route (backward compatibility)
if target_server.route is None:
target_server.set_route(
Route(destination=target_server))
# process routes whose proxy_server is not me
if self.server.id != new_route.get('proxy_server_id'):
# check If I reach destination
if target_server.route.cost is not None:
if new_route.get('cost') is None:
# likely proxy does not reach but I reach it. It might be shutdown unexpectedly?
if target_server.route.cost == 0:
# check if I still have it as a neighbour
route = check_gates(target_server)
else:
if target_server.route.proxy_server == likely_proxy_server:
route = RouteContainer(
None, None, None)
else:
# check if I still have access through my proxy
cost, time = ntwrk.ping(
target_server,
retries=1,
timeout=20,
session=self.session)
if cost == target_server.route.cost:
# still a valid route
route = target_server.route
elif cost is None:
route = RouteContainer(
None, None, None)
else:
route = RouteContainer(
target_server.route.
proxy_server, None, cost)
if isinstance(route, RouteContainer):
# gate changed
changed_routes[target_server] = route
target_server.set_route(route)
elif route is None:
# no route to host. I've lost contact too
changed_routes[
target_server] = RouteContainer(
None, None, None)
target_server.set_route(
changed_routes[target_server])
else:
# still a valid route. Send route to likely_proxy_server to tell it I have access
resp = ntwrk.patch(
likely_proxy_server,
'api_1_0.routes',
json=dict(
server_id=str(self.server.id),
route_list=[
target_server.route.to_json()
]),
auth=get_root_auth(),
timeout=5)
if not resp.ok:
self.logger.info(
f'Unable to send route to {likely_proxy_server}: {resp}'
)
elif target_server.route.proxy_server is not None and \
target_server.route.proxy_server == likely_proxy_server:
# my proxy is telling me the route has changed
rc = RouteContainer(likely_proxy_server, None,
new_route.get('cost') + 1)
target_server.set_route(rc)
changed_routes.update({target_server: rc})
elif new_route.get(
'cost') + 1 < target_server.route.cost:
# if new route has less cost than actual route, take it as my new route
rc = RouteContainer(likely_proxy_server, None,
new_route.get('cost') + 1)
target_server.set_route(rc)
changed_routes.update({target_server: rc})
else:
# me does not reaches target_server
# if new route reaches the destination take it as a new one
if (new_route.get('cost') == 0
and new_route.get('gate_id') is not None
) or (new_route.get('cost') is not None
and proxy_server is not None):
rc = RouteContainer(likely_proxy_server, None,
new_route.get('cost') + 1)
else:
# neither my route and the new route has access to the destination
rc = RouteContainer(None, None, None)
if target_server.route and (
rc.proxy_server !=
target_server.route.proxy_server
or rc.gate != target_server.route.gate
or rc.cost != target_server.route.cost):
target_server.set_route(rc)
changed_routes.update({target_server: rc})
else:
# target_server reached through me as a proxy from likely_proxy
pass
query = self.session.query(Route).filter(
Route.proxy_server_id.in_([
s.id for s, r in changed_routes.items() if r.cost is None
]))
rc = RouteContainer(None, None, None)
for route in query.all():
route.set_route(rc)
changed_routes[route.destination] = rc
self.logger.debug(
f"New routes processed from {likely_proxy_server.name}: {json.dumps(debug_new_routes, indent=2)}"
)
# if changed_routes:
# Parameter.set('routing_last_refresh', get_now())
except errors.InvalidRoute as e:
debug_new_routes = []
routes.sort(key=lambda x: x.get('cost') or MAX_COST, reverse=True)
for new_route in routes:
target_server = self.session.query(Server).get(
new_route.get('destination_id'))
proxy_server = self.session.query(Server).get(
new_route.get('proxy_server_id'))
gate = self.session.query(Gate).get(new_route.get('gate_id'))
dest_name = getattr(target_server, 'name',
new_route.get('destination_id'))
proxy_name = getattr(proxy_server, 'name',
new_route.get('proxy_server_id'))
gate_str = str(gate) if gate else new_route.get('gate_id')
cost = new_route.get('cost')
if gate_str and proxy_name:
gate_str = gate_str + '*' + proxy_name
debug_new_routes.append(
f'{dest_name} -> {gate_str or proxy_name} / {cost}')
self.logger.exception(
"Error setting routes from following data: " +
json.dumps(debug_new_routes, indent=4))
return changed_routes
async def _async_refresh_route_table(
self,
discover_new_neighbours=False,
check_current_neighbours=False,
max_num_discovery=None) -> t.Dict[Server, RouteContainer]:
"""Gets route tables of all neighbours and updates its own table based on jump weights.
Needs a Flask App Context to run.
Parameters
----------
discover_new_neighbours:
tries to discover new neighbours
check_current_neighbours:
checks if current neighbours are still neighbours
max_num_discovery:
maximum number of possible nodes to check as neighbour
Returns
-------
None
"""
self.logger.debug('Refresh Route Table')
neighbours = Server.get_neighbours(session=self.session)
not_neighbours = Server.get_not_neighbours(session=self.session)
changed_routes: t.Dict[Server, RouteContainer] = {}
not_neighbours_anymore = []
new_neighbours = []
aws = []
if check_current_neighbours:
if neighbours:
self.logger.debug(f"Checking current neighbours: " +
', '.join([str(s) for s in neighbours]))
aws.append(
_async_set_current_neighbours(neighbours, changed_routes))
else:
self.logger.debug(f"No neighbour to check")
if discover_new_neighbours:
if not_neighbours[:max_num_discovery]:
rs = list(not_neighbours)
random.shuffle(rs)
target = rs[:max_num_discovery]
target.sort(key=lambda s: s.name)
self.logger.debug(
f"Checking new neighbours{f' (limited to {max_num_discovery})' if max_num_discovery else ''}: "
+ ', '.join([str(s) for s in target]))
aws.append(
_async_discover_new_neighbours(target, changed_routes))
else:
self.logger.debug("No new neighbours to check")
res = await asyncio.gather(*aws, return_exceptions=False)
if check_current_neighbours and neighbours:
not_neighbours_anymore = res.pop(0)
if not_neighbours_anymore:
self.logger.info(
f"Lost direct connection to the following nodes: " +
', '.join([str(s) for s in not_neighbours_anymore]))
if discover_new_neighbours and not_neighbours[:max_num_discovery]:
new_neighbours = res.pop(0)
if new_neighbours:
self.logger.info(f'New neighbours found: ' +
', '.join([str(s) for s in new_neighbours]))
else:
self.logger.debug("No new neighbours found")
# remove routes whose proxy_server is a node that is not a neighbour
query = self.session.query(Route).filter(
Route.proxy_server_id.in_([
s.id for s in list(
set(not_neighbours).union(set(not_neighbours_anymore)))
]))
rc = RouteContainer(None, None, None)
for route in query.all():
route.set_route(rc)
changed_routes[route.destination] = rc
self.session.commit()
# update neighbour lis
neighbours = list(
set(neighbours).union(set(new_neighbours)) -
set(not_neighbours_anymore))
if neighbours:
self.logger.debug(
f"Getting routing tables from {', '.join([str(s) for s in neighbours])}"
)
responses = await asyncio.gather(*[
ntwrk.async_get(server, 'api_1_0.routes', auth=get_root_auth())
for server in neighbours
])
cr = self._route_table_merge(dict(zip(neighbours, responses)))
changed_routes.update(cr)
return changed_routes
def _route_table_merge(self, data: t.Dict[Server, ntwrk.Response]):
changed_routes: t.Dict[Server, RouteContainer] = {}
temp_table_routes: t.Dict[uuid.UUID, t.List[RouteContainer]] = {}
for s, resp in data.items():
if resp.code == 200:
server_id = resp.msg.get(
'server_id', None) or resp.msg.get('server').get('id')
likely_proxy_server_entity = self.session.query(Server).get(
server_id)
for route_json in resp.msg['route_list']:
route_json = convert(route_json)
if route_json.destination_id != self.server.id \
and route_json.proxy_server_id != self.server.id \
and route_json.gate_id not in [g.id for g in self.server.gates]:
if route_json.destination_id not in temp_table_routes:
temp_table_routes.update(
{route_json.destination_id: []})
if route_json.cost is not None:
route_json.cost += 1
route_json.proxy_server_id = likely_proxy_server_entity.id
route_json.gate_id = None
temp_table_routes[
route_json.destination_id].append(
RouteContainer(
likely_proxy_server_entity.id, None,
route_json.cost))
elif route_json.cost is None:
# remove a routing if gateway cannot reach the destination
temp_table_routes[
route_json.destination_id].append(
RouteContainer(route_json.proxy_server_id,
None, None))
else:
self.logger.error(
f"Error while connecting with {s}. Error: {resp}")
# Select new routes based on neighbour routes
neighbour_ids = [
s.id for s in Server.get_neighbours(session=self.session)
]
for destination_id in filter(lambda s: s not in neighbour_ids,
temp_table_routes.keys()):
route = self.session.query(Route).filter_by(
destination_id=destination_id).one_or_none()
if not route:
server = self.session.query(Server).get(destination_id)
if not server:
continue
else:
route = Route(destination=server)
temp_table_routes[destination_id].sort(
key=lambda x: x.cost or MAX_COST)
if len(temp_table_routes[destination_id]) > 0:
min_route = temp_table_routes[destination_id][0]
proxy_server: Server = self.session.query(Server).get(
min_route.proxy_server)
cost = min_route.cost
if route.proxy_server != proxy_server or route.cost != cost:
rc = RouteContainer(proxy_server, None, cost)
route.set_route(rc)
changed_routes[route.destination] = rc
self.session.add(route)
data = {}
for server, temp_route in changed_routes.items():
data.update({
str(server): {
'proxy_server': str(temp_route.proxy_server),
'gate': str(temp_route.gate),
'cost': str(temp_route.cost)
}
})
return changed_routes