def __init__(self, top_man, overlay_id, node_id, max_wrkld):
self._current_adj_list = ConnEdgeAdjacenctList(overlay_id, node_id)
self._pending_adj_list = None
self._negotiated_edges = {}
self._refresh_in_progress = 0
self._max_concurrent_wrkload = max_wrkld
#self._lock = threading.Lock()
self._top = top_man
self._net_ops = None
self.dbg_netop_max_pri = 0
def build_adj_list(self, transition_adj_list):
adj_list = ConnEdgeAdjacenctList(
self.overlay_id, self._node_id,
dict(MaxSuccessors=self._max_successors,
MaxLongDistEdges=self._max_ldl_cnt))
self._build_enforced(adj_list)
if not self._manual_topo:
self._build_successors(adj_list)
self._build_long_dist_links(adj_list, transition_adj_list)
adj_list.validate()
return adj_list
def build_adj_list(self, ):
"""
Generates a new adjacency list from the list of available peers
"""
adj_list = ConnEdgeAdjacenctList(self.overlay_id, self._node_id)
for peer_id in self._peers:
if self._enforced and peer_id in self._enforced:
ce = ConnectionEdge(peer_id)
ce.type = "CETypeEnforced"
adj_list.conn_edges[peer_id] = ce
elif not self._manual_topo and self._node_id < peer_id:
ce = ConnectionEdge(peer_id)
ce.type = "CETypeSuccessor"
adj_list.conn_edges[peer_id] = ce
return adj_list
def build_adj_list_ata(self,):
"""
Generates a new adjacency list from the list of available peers
"""
adj_list = ConnEdgeAdjacenctList(self.overlay_id, self._node_id,
self._max_successors, self._max_ldl_cnt, self._max_ond)
for peer_id in self._peers:
if self._enforced and peer_id in self._enforced:
ce = ConnectionEdge(peer_id)
ce.edge_type = "CETypeEnforced"
adj_list.add_connection_edge(ce)
elif not self._manual_topo and self._node_id < peer_id:
ce = ConnectionEdge(peer_id)
ce.edge_type = "CETypeSuccessor"
adj_list.add_connection_edge(ce)
return adj_list
def build_adj_list(self,
peers,
transition_adj_list,
request_list=None,
relink=False):
self._relink = relink
self._prep(peers)
if request_list is None:
request_list = []
adj_list = ConnEdgeAdjacenctList(self.overlay_id, self._node_id,
self._max_successors,
self._max_ldl_cnt, self._max_ond)
self._build_enforced(adj_list)
if not self._manual_topo:
self._build_successors(adj_list, transition_adj_list)
self._build_long_dist_links(adj_list, transition_adj_list)
self._build_ondemand_links(adj_list, transition_adj_list,
request_list)
for _, ce in adj_list.conn_edges.items():
assert ce.edge_state == "CEStateInitialized", "Invalid CE edge state, CE={}".format(
ce)
return adj_list
def build_adj_list_ata(self, ):
"""
Generates a new adjacency list from the list of available peers
"""
adj_list = ConnEdgeAdjacenctList(self.overlay_id, self._node_id,
self._max_successors,
self._max_ldl_cnt, self._max_ond)
for peer_id in self._peers:
if self._enforced_edges and peer_id in self._enforced_edges:
ce = ConnectionEdge(peer_id)
ce.edge_type = "CETypeEnforced"
adj_list.add_conn_edge(ce)
elif not self._manual_topo and self._node_id < peer_id:
ce = ConnectionEdge(peer_id)
ce.edge_type = "CETypeSuccessor"
adj_list.add_conn_edge(ce)
return adj_list
class NetworkBuilder():
_DEL_RETRY_INTERVAL = 10
"""description of class"""
def __init__(self, top_man, overlay_id, node_id, max_wrkld):
self._current_adj_list = ConnEdgeAdjacenctList(overlay_id, node_id)
self._pending_adj_list = None
self._negotiated_edges = {}
self._refresh_in_progress = 0
self._max_concurrent_wrkload = max_wrkld
#self._lock = threading.Lock()
self._top = top_man
self._net_ops = None
self.dbg_netop_max_pri = 0
def __repr__(self):
state = "current_adj_list=%s, pending_adj_list=%s, negotiated_edges=%s, "\
"refresh_in_progress=%s, _max_concurrent_wrkload=%s" % \
(self._current_adj_list, self._pending_adj_list, self._negotiated_edges,
self._refresh_in_progress, self._max_concurrent_wrkload)
return state
@property
def is_ready(self):
"""
Is the NetworkBuilder ready for a new NetGraph? This means all the entries in the
pending adj list has been cleared.
"""
return self._is_ready()
def _is_ready(self):
return not bool(self._net_ops)
def _is_max_concurrent_workload(self):
return self._refresh_in_progress >= self._max_concurrent_wrkload
def get_adj_list(self):
return deepcopy(self._current_adj_list)
def refresh(self, net_graph=None):
"""
Transitions the overlay network overlay to the desired state specified by pending
adjacency list.
"""
self._top.log("LOG_DEBUG", "New net graph: %s", str(net_graph))
#assert ((self._is_ready() and bool(net_graph)) or
# (not self._is_ready() and not bool(net_graph))),\
# "Netbuilder is not ready for a new net graph"
self.dbg_netop_max_pri = 0
if net_graph and self._is_ready():
self._pending_adj_list = net_graph
self._current_adj_list.max_successors = net_graph.max_successors
self._current_adj_list.max_ldl = net_graph.max_ldl
self._current_adj_list.max_ondemand = net_graph.max_ondemand
self._current_adj_list.update_closest()
self._net_ops = NetworkOperations(self._current_adj_list,
self._pending_adj_list)
self._net_ops.diff()
self._top.log("LOG_DEBUG", "net_op=%s", str(self._net_ops))
self.process_net_ops()
def update_edge_state(self, event):
"""
Updates the connection edge's current state based on the provided event. The number of CEs
not in the EdgeState CEStateConnected is used to limit the number of edges being
constructed concurrently.
"""
peer_id = event["PeerId"]
edge_id = event["TunnelId"]
overlay_id = event["OverlayId"]
if event["UpdateType"] == "LnkEvAuthorized":
self._add_incoming_auth_conn_edge(peer_id)
elif event["UpdateType"] == "LnkEvDeauthorized":
ce = self._current_adj_list[peer_id]
assert ce.edge_state == "CEStateAuthorized", "Deauth CE={0}".format(
ce)
ce.edge_state = "CEStateDeleting"
del self._current_adj_list[peer_id]
del self._pending_adj_list[peer_id]
self._refresh_in_progress -= 1
elif event["UpdateType"] == "LnkEvCreating":
conn_edge = self._current_adj_list.conn_edges.get(peer_id, None)
conn_edge.edge_state = "CEStateCreated"
elif event["UpdateType"] == "LnkEvConnected":
self._current_adj_list[peer_id].edge_state = "CEStateConnected"
self._current_adj_list[peer_id].connected_time = \
event["ConnectedTimestamp"]
del self._pending_adj_list[peer_id]
self._refresh_in_progress -= 1
elif event["UpdateType"] == "LnkEvDisconnected":
# the local topology did not request removal of the connection
self._top.log(
"LOG_DEBUG",
"CEStateDisconnected event recvd peer_id: %s, edge_id: %s",
peer_id, edge_id)
self._current_adj_list[peer_id].edge_state = "CEStateDisconnected"
self._refresh_in_progress += 1
self._top.top_remove_edge(overlay_id, peer_id)
elif event["UpdateType"] == "LnkEvRemoved":
self._current_adj_list[peer_id].edge_state = "CEStateDeleting"
del self._current_adj_list[peer_id]
del self._pending_adj_list[peer_id]
self._refresh_in_progress -= 1
elif event["UpdateType"] == "RemoveEdgeFailed":
# leave the node in the adj list and marked for removal to be retried.
# the retry occurs too quickly and causes too many attempts before it succeeds
self._refresh_in_progress -= 1
self._current_adj_list[peer_id].created_time = \
time.time() + NetworkBuilder._DEL_RETRY_INTERVAL
else:
self._top.log("LOG_WARNING",
"Invalid UpdateType specified for event")
assert self._refresh_in_progress >= 0, "refresh in progress is negative {}"\
.format(self._refresh_in_progress)
def _update_conn_edge(self, conn_edge):
peer_id = conn_edge.peer_id
self._current_adj_list[peer_id].edge_type = conn_edge.edge_type
def _remove_conn_edge(self, conn_edge):
overlay_id = self._current_adj_list.overlay_id
if conn_edge.edge_state == "CEStateConnected":
self._refresh_in_progress += 1
conn_edge.edge_state = "CEStateDeleting"
self._top.top_remove_edge(overlay_id, conn_edge.peer_id)
def _create_conn_edge(self, conn_edge):
conn_edge.edge_state = "CEStatePreAuth"
self._current_adj_list[conn_edge.peer_id] = conn_edge
self._negotiate_new_edge(conn_edge.edge_id, conn_edge.edge_type,
conn_edge.peer_id, conn_edge)
def process_net_ops(self):
for nop in self._net_ops:
assert self.dbg_netop_max_pri <= nop.op_priority, \
"Invalid Netop priority ordering max:{0} pri:{1}".format(self.dbg_netop_max_pri,
nop.op_priority)
self.dbg_netop_max_pri = nop.op_priority
if self._is_max_concurrent_workload():
return
if nop.op_type == OpType[2]:
self._update_conn_edge(nop.conn_edge)
elif nop.op_type == OpType[1]:
self._remove_conn_edge(nop.conn_edge)
elif nop.op_type == OpType[0]:
self._create_conn_edge(nop.conn_edge)
def _negotiate_new_edge(self, edge_id, edge_type, peer_id, conn_edge):
""" Role A1 """
self._refresh_in_progress += 1
self._negotiated_edges[peer_id] = conn_edge
olid = self._current_adj_list.overlay_id
nid = self._current_adj_list.node_id
er = EdgeRequest(overlay_id=olid,
edge_id=edge_id,
edge_type=edge_type,
recipient_id=peer_id,
initiator_id=nid)
self._top.top_send_negotiate_edge_req(er)
def _resolve_request_collision(self, edge_req):
nid = self._top.node_id
peer_id = edge_req.initiator_id
ce = self._current_adj_list[peer_id]
edge_state = ce.edge_state
edge_resp = None
if edge_state in ("CEStateAuthorized", "CEStateCreated",
"CEStateConnected"):
msg = "E1 - A valid edge already exists. TunnelId={0}"\
.format(self._current_adj_list[peer_id].edge_id[:7])
edge_resp = EdgeResponse(is_accepted=False, data=msg)
self._top.log("LOG_DEBUG", msg)
elif edge_state == "CEStateInitialized":
edge_resp = EdgeResponse(is_accepted=True,
data="Precollision edge permitted")
del self._current_adj_list[peer_id]
del self._pending_adj_list[peer_id]
del self._negotiated_edges[peer_id]
elif edge_state == "CEStatePreAuth" and nid < edge_req.initiator_id:
msg = "E2 - Node {0} superceeds edge request due to collision, "\
"edge={1}".format(nid, self._current_adj_list[peer_id].edge_id[:7])
edge_resp = EdgeResponse(is_accepted=False, data=msg)
self._top.log("LOG_DEBUG", msg)
elif edge_state == "CEStatePreAuth" and nid > edge_req.initiator_id:
ce.edge_type = ng.transpose_edge_type(edge_req.edge_type)
ce.edge_id = edge_req.edge_id
msg = "E0 - Node {2} accepts edge collision override. CE:{0} remapped -> edge:{1}"\
.format(ce, edge_req.edge_id[:7], nid)
edge_resp = EdgeResponse(is_accepted=True, data=msg)
self._top.log("LOG_DEBUG", msg)
else:
edge_resp = EdgeResponse(False, "E6 - Request colides with an edge being destroyed."\
"Try later")
assert bool(edge_resp), "NetBuilder={0}".format(self)
return edge_resp
def negotiate_incoming_edge(self, edge_req):
""" Role B1 """
self._top.log("LOG_DEBUG", "Rcvd EdgeRequest=%s", str(edge_req))
edge_resp = None
peer_id = edge_req.initiator_id
if peer_id in self._current_adj_list:
edge_resp = self._resolve_request_collision(edge_req)
elif edge_req.edge_type == "CETypeSuccessor":
edge_resp = EdgeResponse(is_accepted=True,
data="Successor edge permitted")
elif edge_req.edge_type == "CETypeEnforced":
edge_resp = EdgeResponse(is_accepted=True,
data="Enforced edge permitted")
elif edge_req.edge_type == "CETypeOnDemand":
edge_resp = EdgeResponse(is_accepted=True,
data="On-demand edge permitted")
elif not self._current_adj_list.is_threshold_ildl():
edge_resp = EdgeResponse(is_accepted=True,
data="Any edge permitted")
else:
edge_resp = EdgeResponse(is_accepted=False,
data="E5 - Too many existing edges.")
if edge_resp.is_accepted and edge_resp.data[:2] != "E0":
et = ng.transpose_edge_type(edge_req.edge_type)
ce = ConnectionEdge(peer_id=peer_id,
edge_id=edge_req.edge_id,
edge_type=et)
ce.edge_state = "CEStatePreAuth"
self._negotiated_edges[peer_id] = ce
self._top.log("LOG_DEBUG",
"New CE=%s added to negotiated_edges=%s", str(ce),
str(self._negotiated_edges))
return edge_resp
def _add_incoming_auth_conn_edge(self, peer_id):
""" Role B2 """
self._refresh_in_progress += 1
ce = self._negotiated_edges.pop(peer_id)
ce.edge_state = "CEStateAuthorized"
self._current_adj_list.add_conn_edge(ce)
def complete_edge_negotiation(self, edge_nego):
""" Role A2 """
self._top.log("LOG_DEBUG", "EdgeNegotiate=%s", str(edge_nego))
if edge_nego.recipient_id not in self._current_adj_list and \
edge_nego.recipient_id not in self._negotiated_edges:
self._top.log("LOG_ERROR", "Peer Id from edge negotiation not in current adjacency " \
" list or _negotiated_edges. The transaction has been discarded.")
return
peer_id = edge_nego.recipient_id
edge_id = edge_nego.edge_id
ce = self._negotiated_edges.get(edge_nego.recipient_id,
None) # do not pop here, E0 needed
if not ce:
return # OK - Collision override occurred, CE was popped in role B2 (above). Completion
# order can vary, in other case handled below.
if not edge_nego.is_accepted:
self._refresh_in_progress -= 1
# if E2 (request superceeded) do nothing here. The corresponding CE instance will
# be converted in resolve_collision_request().
if edge_nego.data[:2] != "E2":
ce.edge_state = "CEStateDeleting"
self._negotiated_edges.pop(ce.peer_id)
del self._pending_adj_list[peer_id]
del self._current_adj_list[ce.peer_id]
else:
if ce.edge_id != edge_nego.edge_id:
self._top.log("LOG_ERROR", "EdgeNego parameters does not match current " \
"adjacency list, The transaction has been discarded.")
ce.edge_state = "CEStateDeleting"
self._negotiated_edges.pop(ce.peer_id)
del self._pending_adj_list[peer_id]
del self._current_adj_list[ce.peer_id]
self._refresh_in_progress -= 1
else:
ce.edge_state = "CEStateAuthorized"
self._negotiated_edges.pop(ce.peer_id)
self._top.top_add_edge(self._current_adj_list.overlay_id,
peer_id, edge_id)
def __init__(self, top_man, overlay_id, node_id):
self._current_edges = ConnEdgeAdjacenctList(overlay_id, node_id)
self._refresh_in_progress = 0
self._pending_edges = None
self._lock = threading.Lock()
self._top = top_man
class NetworkBuilder():
"""description of class"""
def __init__(self, top_man, overlay_id, node_id):
self._current_adj_list = ConnEdgeAdjacenctList(overlay_id, node_id)
self._refresh_in_progress = 0
self._pending_adj_list = None
self._lock = threading.Lock()
self._top = top_man
def is_ready(self):
with self._lock:
if self._refresh_in_progress < 0:
raise ValueError(
"A precondition violation occurred. The refresh reference count"
" is negative {}".format(self._refresh_in_progress))
return self._refresh_in_progress == 0
def get_adj_list(self):
with self._lock:
return deepcopy(self._current_adj_list)
def refresh(self, net_graph=None):
"""
Updates the networks connections. Invoked on different threads: 1) Periodically without
parameters to last to last provided network graph, 2) attempt to refresh now or schedule
the provide graph for refresh.
"""
with self._lock:
self._top.top_log("New net graph:{0}\nCurrent adj list:{1}".format(
net_graph, self._current_adj_list))
if self._pending_adj_list:
self._top.top_log("Pending adj list:{0}".format(
self._pending_adj_list))
"""
This conditon is expected to be met on the timer invocation when no net_graph is
supplied but when there is _pending_edges waiting to be applied.
"""
if self._refresh_in_progress == 0 and not net_graph and self._pending_adj_list:
self._update_net_connections()
return
"""
Overwrite any previous pending_edges as we are only interested in the most recent one.
"""
if net_graph:
self._pending_adj_list = net_graph
"""
To minimize network disruption wait until a previous sync operation is completed before
starting a new one.
"""
if self._refresh_in_progress > 0:
return
"""
Attempt to sync the network state to the pending net graph.
"""
if self._pending_adj_list:
self._update_net_connections()
def on_connection_update(self, connection_event):
"""
Updates the connection edge's current state based on the provided event. This is the
completion for a create or remove connection request to Link Manager.
"""
peer_id = connection_event["PeerId"]
link_id = connection_event["LinkId"]
overlay_id = connection_event["OverlayId"]
with self._lock:
if connection_event["UpdateType"] == "CREATING":
conn_edge = self._current_adj_list.conn_edges.get(
peer_id, None)
if not conn_edge:
# this happens when the neighboring peer initiates the connection bootstrap
self._refresh_in_progress += 1
conn_edge = ConnectionEdge(peer_id, "CETypePredecessor")
self._current_adj_list.conn_edges[peer_id] = conn_edge
conn_edge.edge_state = "CEStateCreated"
conn_edge.link_id = link_id
elif connection_event["UpdateType"] == "REMOVED":
self._current_adj_list.conn_edges.pop(peer_id, None)
self._refresh_in_progress -= 1
elif connection_event["UpdateType"] == "CONNECTED":
self._current_adj_list.conn_edges[
peer_id].edge_state = "CEStateConnected"
self._current_adj_list.conn_edges[peer_id].connected_time = \
connection_event["ConnectedTimestamp"]
self._refresh_in_progress -= 1
elif connection_event["UpdateType"] == "DISCONNECTED":
# the local topology did not request removal of the connection
self._top.top_log(
"CEStateDisconnected event recvd peer_id: {0}, link_id: {1}"
.format(peer_id, link_id))
self._current_adj_list.conn_edges[
peer_id].edge_state = "CEStateDisconnected"
self._refresh_in_progress += 1
self._top.top_remove_edge(overlay_id, peer_id)
elif connection_event["UpdateType"] == "RemoveEdgeFailed":
# leave the node in the adj list and marked for removal to be retried.
self._refresh_in_progress -= 1
else:
self._top.top_log(
"Logger", "LOG_WARNING",
"Invalid UpdateType specified for connection update")
def _mark_edges_for_removal(self):
""" Anything edge the set (Active - Pending) is marked for deletion """
for peer_id in self._current_adj_list.conn_edges:
if (peer_id not in self._pending_adj_list.conn_edges
and self._current_adj_list.conn_edges[peer_id].edge_type !=
"CETypePredecessor"):
self._current_adj_list.conn_edges[
peer_id].marked_for_delete = True
def _remove_edges(self):
""" Minimize churn by removing a single connection per refresh. Only initiate removal of a
successor edge if a replacement has been previously connected.
"""
overlay_id = self._current_adj_list.overlay_id
conn_succ = self._current_adj_list.filter("CETypeSuccessor",
"CEStateConnected")
num_conn_succ = len(conn_succ)
for peer_id in self._current_adj_list.conn_edges:
if self._current_adj_list.conn_edges[peer_id].marked_for_delete:
if (self._current_adj_list.conn_edges[peer_id].edge_type !=
"CETypeSuccessor" or
(self._current_adj_list.conn_edges[peer_id].edge_type
== "CETypeSuccessor" and
num_conn_succ >= self._current_adj_list.max_successors)):
self._refresh_in_progress += 1
self._top.top_remove_edge(overlay_id, peer_id)
return
def _create_new_edges(self):
""" Any edge in set (Pending - Active) is created and added to Active """
overlay_id = self._current_adj_list.overlay_id
for peer_id in self._pending_adj_list.conn_edges:
if not peer_id in self._current_adj_list.conn_edges:
self._current_adj_list.conn_edges[peer_id] = \
self._pending_adj_list.conn_edges[peer_id]
if self._current_adj_list.conn_edges[
peer_id].edge_state == "CEStateUnknown":
self._refresh_in_progress += 1
self._top.top_add_edge(overlay_id, peer_id)
else:
# Existing edges in both Active and Pending are updated in place. Only the marked
# for delete and edge type fields can be meaningfully changed in this case.
self._current_adj_list.conn_edges[peer_id].marked_for_delete = \
self._pending_adj_list.conn_edges[peer_id].marked_for_delete
self._current_adj_list.conn_edges[peer_id].edge_type = \
self._pending_adj_list.conn_edges[peer_id].edge_type
def _update_net_connections(self):
"""
Sync the network state by determining the difference between the active and pending net
graphs. Create new successors edges before removing existing ones.
"""
if self._current_adj_list.overlay_id != self._pending_adj_list.overlay_id:
raise ValueError(
"Overlay ID mismatch adj lists, active:{0}, pending:{1}".
format(self._current_adj_list.overlay_id,
self._pending_adj_list.overlay_id))
self._mark_edges_for_removal()
self._create_new_edges()
self._pending_adj_list = None
self._remove_edges()