def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._maintenance_mode = BOOTSTRAP_MODE self._pinged_q_rnodes = {} # questionable nodes which have been # recently pinged self._maintenance_tasks = [ self._refresh_stale_bucket, #self._ping_a_staled_rnode, # self._ping_a_query_received_node, # self._ping_a_found_node, ]
def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [ self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node ]
def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [ self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
def __init__(self, my_node, msg_f, bootstrapper): self.my_node = my_node self.msg_f = msg_f self.bootstrapper = bootstrapper self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = FILL_BUCKETS self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [ self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS self._num_timeouts_in_a_row = 0
def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._maintenance_mode = BOOTSTRAP_MODE self._pinged_q_rnodes = {} # questionable nodes which have been # recently pinged self._maintenance_tasks = [self._refresh_stale_bucket, #self._ping_a_staled_rnode, # self._ping_a_query_received_node, # self._ping_a_found_node, ]
def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery(my_node.id, my_node.id) self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node]
def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node self.bootstrapper = bootstrap.OverlayBootstrapper(my_node.id, bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ]
def __init__(self, my_node, msg_f, bootstrapper): self.my_node = my_node self.msg_f = msg_f self.bootstrapper = bootstrapper self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = FILL_BUCKETS self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS self._num_timeouts_in_a_row = 0
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery(my_node.id, my_node.id) self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except StopIteration: maintenance_lookup_target = self.my_node.id self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): starting_index = self._next_stale_maintenance_index result = None while not result: sbucket = self.table.get_sbucket(self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = (self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: return Query(self.find_closest_msg, node_) elif random.choice((False, True)): return Query(self.find_closest_msg, node_) target_log_distance = self.table.find_next_bucket_with_room_index(node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return Query(message.OutgoingFindNodeQuery(self.my_node.id, target), node_) else: return Query(self.ping_msg, node_) def on_query_received(self, node_): log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_query_received(rnode) return if m_bucket.there_is_room(): self._query_received_queue.add(node_, log_distance) return worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_response_received(rnode, rtt) return rnode = r_bucket.get_rnode(node_) if rnode: self._update_rnode_on_response_received(rnode, rtt) if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return current_time = time.time() rnode_to_be_replaced = None for rnode in reversed(m_bucket.rnodes): rnode_age = current_time - rnode.bucket_insertion_ts if rtt < rnode.rtt * (1 - rnode_age / 7200): rnode_to_be_replaced = rnode break if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) def on_error_received(self, node_addr): pass def on_timeout(self, node_): if not node_.id: return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._replacement_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: r_bucket.remove(worst_rnode) r_bucket.add(rnode) rnode = r_bucket.get_rnode(node_) if rnode: self._update_rnode_on_timeout(rnode) def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): rnode.rtt = rtt current_time = time.time() if rnode.in_quarantine: rnode.in_quarantine = rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) # maintenance variables self._maintenance_mode = BOOTSTRAP_MODE self._pinged_q_rnodes = {} # questionable nodes which have been # recently pinged self._maintenance_tasks = [ self._refresh_stale_bucket, #self._ping_a_staled_rnode, # self._ping_a_query_received_node, # self._ping_a_found_node, ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = FIND_NODES_MODE return (10, [], maintenance_lookup_target) else: maintenance_lookup_target = self._refresh_stale_bucket() return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _refresh_stale_bucket(self): maintenance_lookup_target = None current_time = time.time() for i in xrange(self.table.lowest_index, NUM_BUCKETS): sbucket = self.table.get_sbucket(i) m_bucket = sbucket.main if not m_bucket: continue inactivity_time = current_time - m_bucket.last_changed_ts if inactivity_time > REFRESH_PERIOD: # print time.time(), '>>>>>>>>>>>>>>> refreshing bucket %d after %f secs' % ( # i, inactivity_time) maintenance_lookup_target = self.my_node.id.generate_close_id( i) m_bucket.last_changed_ts = current_time return maintenance_lookup_target self._maintenance_mode = NORMAL_MODE return None def _get_maintenance_query(self, node_): return Query(self.ping_msg, node_) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' if self._maintenance_mode != NORMAL_MODE: return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket. Just add the new node. rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_query_received(rnode) return # No room in the main routing table # Check whether there is a bad node to be replaced. bad_rnode = self._pop_bad_rnode(m_bucket) if bad_rnode: # We have a bad node in the bucket. Replace it with the new node. rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self._update_rnode_on_query_received(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 0 return # No bad nodes. Check for questionable nodes q_rnodes = self._get_questionable_rnodes(m_bucket) queries_to_send = [] # if q_rnodes: # print time.time(), '-----pinging questionable nodes in', # print log_distance # print q_rnodes for q_rnode in q_rnodes: # Ping questinable nodes to check whether they are still alive. # (0 timeouts so far, candidate node) c_rnode = node_.get_rnode(log_distance) self._update_rnode_on_query_received(c_rnode) self._pinged_q_rnodes[q_rnode] = [0, c_rnode] queries_to_send.append(Query(self.ping_msg, q_rnode)) return queries_to_send def on_response_received(self, node_, rtt, nodes): log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() if node_ in self._pinged_q_rnodes: # This node is questionable. This response proves that it is # alive. Remove it from the questionable dict. del self._pinged_q_rnodes[node_] return # The node is not in main if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() return # The main bucket is full # if there is a bad node inside the bucket, # replace it with the sending node_ bad_rnode = self._pop_bad_rnode(m_bucket) if bad_rnode: rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() self.table.update_lowest_index(log_distance) self.table.num_rnodes += 0 return # There are no bad nodes. Ping questionable nodes (if any) q_rnodes = self._get_questionable_rnodes(m_bucket) queries_to_send = [] for q_rnode in q_rnodes: # (0 timeouts so far, candidate node) c_rnode = node_.get_rnode(log_distance) self._update_rnode_on_response_received(c_rnode, rtt) self._pinged_q_rnodes[q_rnode] = [0, c_rnode] queries_to_send.append(Query(self.ping_msg, q_rnode)) return queries_to_send def _pop_bad_rnode(self, mbucket): for rnode in mbucket.rnodes: if rnode.timeouts_in_a_row() >= 2: mbucket.remove(rnode) return rnode def _get_questionable_rnodes(self, m_bucket): q_rnodes = [] for rnode in m_bucket.rnodes: inactivity_time = time.time() - rnode.last_seen if inactivity_time > REFRESH_PERIOD: q_rnodes.append(rnode) if rnode.num_responses == 0: q_rnodes.append(rnode) return q_rnodes def on_error_received(self, node_): pass def on_timeout(self, node_): if not node_.id: return # This is a bootstrap node (just addr, no id) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if not rnode: # This node is not in the table. Nothing to do here return # The node is in the table. Update it self._update_rnode_on_timeout(rnode) t_strikes, c_rnode = self._pinged_q_rnodes.get(node_, (None, None)) if t_strikes is None: # The node is not being checked by a "questinable ping". return elif t_strikes == 0: # This is the first timeout self._pinged_q_rnodes[node_] = (1, c_rnode) # Let's give it another chance return [Query(self.ping_msg, rnode)] elif t_strikes == 1: # Second timeout. You're a bad node, replace if possible # check if the candidate node is in the routing table log_distance = self.my_node.log_distance(c_rnode) m_bucket = self.table.get_sbucket(log_distance).main c_rnode_in_table = m_bucket.get_rnode(c_rnode) if c_rnode_in_table: print 'questionable node replaced' # replace m_bucket.remove(rnode) m_bucket.add(c_rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 0 def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.rtt = rtt current_time = time.time() #rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = rnode.last_action_ts < (current_time - QUARANTINE_PERIOD) rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._maintenance_mode = BOOTSTRAP_MODE self._pinged_q_rnodes = {} # questionable nodes which have been # recently pinged self._maintenance_tasks = [self._refresh_stale_bucket, #self._ping_a_staled_rnode, # self._ping_a_query_received_node, # self._ping_a_found_node, ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = FIND_NODES_MODE return (10, [], maintenance_lookup_target) else: maintenance_lookup_target = self._refresh_stale_bucket() # print 'beb5', _MAINTENANCE_DELAY[self._maintenance_mode] return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _refresh_stale_bucket(self): maintenance_lookup_target = None current_time = time.time() for i in xrange(NUM_BUCKETS): sbucket = self.table.get_sbucket(i) m_bucket = sbucket.main if not m_bucket: continue inactivity_time = current_time - m_bucket.last_changed_ts if inactivity_time > REFRESH_PERIOD: # print time.time(), '>>>>>>>>>>>>>>> refreshing bucket %d after %f secs' % ( # i, inactivity_time) maintenance_lookup_target = self.my_node.id.generate_close_id( i) m_bucket.last_changed_ts = current_time return maintenance_lookup_target self._maintenance_mode = NORMAL_MODE return None def _get_maintenance_query(self, node_): return message.OutgoingPingQuery(node_, self.my_node.id) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' if self._maintenance_mode != NORMAL_MODE: return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket. Just add the new node. rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.num_rnodes += 1 self._update_rnode_on_query_received(rnode) return # No room in the main routing table # Check whether there is a bad node to be replaced. bad_rnode = self._pop_bad_rnode(m_bucket) if bad_rnode: # We have a bad node in the bucket. Replace it with the new node. rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self._update_rnode_on_query_received(rnode) self.table.num_rnodes += 0 return # No bad nodes. Check for questionable nodes q_rnodes = self._get_questionable_rnodes(m_bucket) queries_to_send = [] for q_rnode in q_rnodes: # Ping questinable nodes to check whether they are still alive. # (0 timeouts so far, candidate node) c_rnode = node_.get_rnode(log_distance) self._update_rnode_on_query_received(c_rnode) self._pinged_q_rnodes[q_rnode] = [0, c_rnode] queries_to_send.append(message.OutgoingPingQuery(node_, self.my_node.id)) return queries_to_send def on_response_received(self, node_, rtt, nodes): log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if rnode: logger.debug('node in main') # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() if node_ in self._pinged_q_rnodes: logger.debug('remove from questionable') rnode.questionable = False # This node is questionable. This response proves that it is # alive. Remove it from the questionable dict. del self._pinged_q_rnodes[node_] return # The node is not in main if m_bucket.there_is_room(): logger.debug('node not in main, there is room') rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() return # The main bucket is full # if there is a bad node inside the bucket, # replace it with the sending node_ logger.debug('node not in main, no room') bad_rnode = self._pop_bad_rnode(m_bucket) if bad_rnode: logger.debug('there is a bad rnode') rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) # No need to update table self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() return # There are no bad nodes. Ping questionable nodes (if any) logger.debug('no bad nodes, ping questionable nodes') q_rnodes = self._get_questionable_rnodes(m_bucket) queries_to_send = [] for q_rnode in q_rnodes: # (0 timeouts so far, candidate node) c_rnode = node_.get_rnode(log_distance) self._update_rnode_on_response_received(c_rnode, rtt) self._pinged_q_rnodes[q_rnode] = [0, c_rnode] queries_to_send.append(message.OutgoingPingQuery(node_, self.my_node.id)) return queries_to_send def _pop_bad_rnode(self, mbucket): for rnode in mbucket.rnodes: if rnode.timeouts_in_a_row() >= 2: mbucket.remove(rnode) return rnode def _get_questionable_rnodes(self, m_bucket): q_rnodes = [] for rnode in m_bucket.rnodes: inactivity_time = time.time() - rnode.last_seen if (inactivity_time > REFRESH_PERIOD or rnode.num_responses == 0): is_questionable = getattr(rnode, 'questionable', False) if not is_questionable: rnode.questionable = True q_rnodes.append(rnode) return q_rnodes def on_error_received(self, node_addr): pass def on_timeout(self, node_): if not node_.id: return [] # This is a bootstrap node (just addr, no id) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return [] # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if not rnode: # This node is not in the table. Nothing to do here return [] # The node is in the table. Update it self._update_rnode_on_timeout(rnode) t_strikes, c_rnode = self._pinged_q_rnodes.get(node_, (None, None)) if t_strikes is None: # The node is not being checked by a "questinable ping". return [] elif t_strikes == 0: # This is the first timeout self._pinged_q_rnodes[node_] = (1, c_rnode) # Let's give it another chance return [message.OutgoingPingQuery(node_, self.my_node.id)] elif t_strikes == 1: # Second timeout. You're a bad node, replace if possible # check if the candidate node is in the routing table log_distance = self.my_node.log_distance(c_rnode) m_bucket = self.table.get_sbucket(log_distance).main c_rnode_in_table = m_bucket.get_rnode(c_rnode) if c_rnode_in_table: print 'questionable node replaced' # replace m_bucket.remove(rnode) m_bucket.add(c_rnode) self.table.num_rnodes += 0 return [] def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.real_rtt = rtt current_time = time.time() #rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = \ rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [ self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): # We try maintenance tasks till one of them actually does work # or we have tried them all (whatever happens first) We loop # in range because I'm going to modify self._maintenance_tasks task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] # This task did do some work. We are done here! break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): # Don't have self._next_stale_maintenance_index lower than # lowest_bucket starting_index = self._next_stale_maintenance_index result = None while not result: # Find a non-empty bucket sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: # No node to be pinged in the whole table. break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: # Extra ping when bootstrapping num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ return def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: # Bootstrap nodes don't have id return Query(self.find_closest_msg, node_) if random.choice((False, True)): # 50% chance to send find_node with my id as target return Query(self.find_closest_msg, node_) # 50% chance to send a find_node to fill up a non-full bucket target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return Query( message.OutgoingFindNodeQuery(self.my_node.id, target), node_) else: # Every bucket is full. We send a ping instead. return Query(self.ping_msg, node_) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket: queue it self._query_received_queue.add(node_, log_distance) return # No room in the main routing table # Add to replacement table (if the bucket is not full) worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) return def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) return # The node is not in main rnode = r_bucket.get_rnode(node_) if rnode: # node in replacement table # let's see whether there is room in the main self._update_rnode_on_response_received(rnode, rtt) #TODO: leave this for the maintenance task if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return # The node is nowhere # Add to main table (if the bucket is not full) #TODO: check whether in replacement_mode if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return # The main bucket is full # Get the worst node in replacement bucket and see whether # it's bad enough to be replaced by node_ worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: # This node is better candidate than worst_rnode r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) return def on_error_received(self, node_): pass def on_timeout(self, node_): if not node_.id: return # This is a bootstrap node (just addr, no id) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: kick it out self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._replacement_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: # Replace worst node in replacement table r_bucket.remove(worst_rnode) r_bucket.add(rnode) # Node is not in main table rnode = r_bucket.get_rnode(node_) if rnode: # Node in replacement table: just update rnode self._update_rnode_on_timeout(rnode) def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.rtt = rtt current_time = time.time() #rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = \ rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes, msg_f): self.my_node = my_node self.bootstrapper = bootstrap.OverlayBootstrapper( my_node.id, bootstrap_nodes, msg_f) self.msg_f = msg_f self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [ self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS self._num_timeouts_in_a_row = 0 def _get_maintenance_lookup(self, lookup_target=None, nodes=[]): if not lookup_target: lookup_target = identifier.RandomId() if not nodes: log_distance = lookup_target.distance(self.my_node.id).log nodes = self.get_closest_rnodes(log_distance, 0, True) return lookup_target, nodes def do_maintenance(self): queries_to_send = [] maintenance_lookup = None maintenance_delay = 0 if self._maintenance_mode == BOOTSTRAP_MODE: (queries_to_send, maintenance_lookup, bootstrap_delay) = self.bootstrapper.do_bootstrap( self.table.num_rnodes) if bootstrap_delay: maintenance_delay = bootstrap_delay else: self._maintenance_mode = FILL_BUCKETS self.bootstrapper.bootstrap_done() elif self._maintenance_mode == FILL_BUCKETS: if self._num_pending_filling_lookups: self._num_pending_filling_lookups -= 1 maintenance_lookup = self._get_maintenance_lookup() else: self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): # We try maintenance tasks till one of them actually does work # or we have tried them all (whatever happens first) We loop # in range because I'm going to modify self._maintenance_tasks task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send.append(self._get_maintenance_query(node_)) # This task did do some work. We are done here! break if self.table.num_rnodes < MIN_RNODES: # Ping more found nodes when routing table has few nodes node_ = self._ping_a_found_node() if node_: queries_to_send.append( self._get_maintenance_query(node_, do_fill_up=True)) if not maintenance_delay: maintenance_delay = _MAINTENANCE_DELAY[self._maintenance_mode] return (maintenance_delay, queries_to_send, maintenance_lookup) def _ping_a_staled_rnode(self): starting_index = self._next_stale_maintenance_index result = None while not result: # Find a non-empty bucket sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: # No node to be pinged in the whole table. break return result def _ping_a_found_node(self): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_, do_fill_up=False): ''' if not node_.id: # Bootstrap nodes don't have id return message.OutgoingFindNodeQuery(node_, self.my_node.id, self.my_node.id, None) ''' if do_fill_up or random.choice((False, True)): # 50% chance to send a find_node to fill up a non-full bucket target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) msg = self.msg_f.outgoing_find_node_query(node_, target, None) else: # Every bucket is full. We send a ping instead. msg = self.msg_f.outgoing_ping_query(node_) else: # 50% chance to send find_node with my id as target msg = self.msg_f.outgoing_find_node_query(node_, self.my_node.id, None) return msg def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' self._num_timeouts_in_a_row = 0 if self.bootstrapper.is_bootstrap_node(node_): return log_distance = self.my_node.distance(node_).log try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement if node_.ip in m_bucket.ips_in_table: rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update rnode self._update_rnode_on_query_received(rnode) # This IP is in the table. Stop here to avoid multiple entries # with the same IP return # Now, consider adding this node to the routing table if m_bucket.there_is_room(): # There is room in the bucket: queue it self._query_received_queue.add(node_, log_distance) return # No room in the main routing table # Add to replacement table (if the bucket is not full) worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) return def on_response_received(self, node_, rtt, nodes): self._num_timeouts_in_a_row = 0 if self.bootstrapper.is_bootstrap_node(node_): return if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.distance(node_).log try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if node_.ip in m_bucket.ips_in_table: rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update rnode self._update_rnode_on_response_received(rnode, rtt) # This IP is in the table. Stop here to avoid multiple entries # with the same IP return # Now, consider adding this node to the routing table rnode = r_bucket.get_rnode(node_) if rnode: # node in replacement table # let's see whether there is room in the main self._update_rnode_on_response_received(rnode, rtt) # TODO: leave this for the maintenance task if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return # The node is nowhere # Add to main table (if the bucket is not full) # TODO: check whether in replacement_mode if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return # The main bucket is full # Let's see whether this node's latency is good current_time = time.time() rnode_to_be_replaced = None m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True) for rnode in m_bucket.rnodes: rnode_age = current_time - rnode.bucket_insertion_ts if rtt < rnode.rtt * (1 - (rnode_age / 7200)): # A rnode can only be replaced when the candidate node's RTT # is shorter by a factor. Over time, this factor # decreases. For instance, when rnode has been in the bucket # for 30 mins (1800 secs), a candidate's RTT must be at most # 25% of the rnode's RTT (ie. two times faster). After two # hours, a rnode cannot be replaced by this method. # print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % ( # rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts) rnode_to_be_replaced = rnode break if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) # No need to update table self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return # Get the worst node in replacement bucket and see whether # it's bad enough to be replaced by node_ worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: # This node is better candidate than worst_rnode r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) return def on_error_received(self, node_addr): # if self.bootstrapper.is_bootstrap_node(node_): # return return def on_timeout(self, node_): self._num_timeouts_in_a_row += 1 if self._num_timeouts_in_a_row > MAX_TIMEOUTS_IN_A_ROW: # stop, do not expell nodes from routing table return [] if self.bootstrapper.is_bootstrap_node(node_): return [] log_distance = self.my_node.distance(node_).log try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return [] # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: kick it out self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._replacement_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: # Replace worst node in replacement table r_bucket.remove(worst_rnode) r_bucket.add(rnode) # Node is not in main table rnode = r_bucket.get_rnode(node_) if rnode: # Node in replacement table: just update rnode self._update_rnode_on_timeout(rnode) return [] def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.rtt = rtt current_time = time.time() # rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = \ rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [ self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except StopIteration: maintenance_lookup_target = self.my_node.id self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): starting_index = self._next_stale_maintenance_index result = None while not result: sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: return Query(self.find_closest_msg, node_) elif random.choice((False, True)): return Query(self.find_closest_msg, node_) target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return Query( message.OutgoingFindNodeQuery(self.my_node.id, target), node_) else: return Query(self.ping_msg, node_) def on_query_received(self, node_): log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_query_received(rnode) return if m_bucket.there_is_room(): self._query_received_queue.add(node_, log_distance) return worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_response_received(rnode, rtt) return rnode = r_bucket.get_rnode(node_) if rnode: self._update_rnode_on_response_received(rnode, rtt) if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return current_time = time.time() rnode_to_be_replaced = None for rnode in reversed(m_bucket.rnodes): rnode_age = current_time - rnode.bucket_insertion_ts if rtt < rnode.rtt * (1 - rnode_age / 7200): rnode_to_be_replaced = rnode break if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) def on_error_received(self, node_addr): pass def on_timeout(self, node_): if not node_.id: return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._replacement_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: r_bucket.remove(worst_rnode) r_bucket.add(rnode) rnode = r_bucket.get_rnode(node_) if rnode: self._update_rnode_on_timeout(rnode) def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): rnode.rtt = rtt current_time = time.time() if rnode.in_quarantine: rnode.in_quarantine = rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): # We try maintenance tasks till one of them actually does work # or we have tried them all (whatever happens first) We loop # in range because I'm going to modify self._maintenance_tasks task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] # This task did do some work. We are done here! break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): # Don't have self._next_stale_maintenance_index lower than # lowest_bucket starting_index = self._next_stale_maintenance_index result = None while not result: # Find a non-empty bucket sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: # No node to be pinged in the whole table. break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: # Extra ping when bootstrapping num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ return def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: # Bootstrap nodes don't have id return Query(self.find_closest_msg, node_) if random.choice((False, True)): # 50% chance to send find_node with my id as target return Query(self.find_closest_msg, node_) # 50% chance to send a find_node to fill up a non-full bucket target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return Query( message.OutgoingFindNodeQuery(self.my_node.id, target), node_) else: # Every bucket is full. We send a ping instead. return Query(self.ping_msg, node_) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket: queue it self._query_received_queue.add(node_, log_distance) return # No room in the main routing table # Add to replacement table (if the bucket is not full) worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) return def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) return # The node is not in main rnode = r_bucket.get_rnode(node_) if rnode: # node in replacement table # let's see whether there is room in the main self._update_rnode_on_response_received(rnode, rtt) #TODO: leave this for the maintenance task if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return # The node is nowhere # Add to main table (if the bucket is not full) #TODO: check whether in replacement_mode if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return # The main bucket is full # Let's see whether this node's latency is good current_time = time.time() rnode_to_be_replaced = None for rnode in reversed(m_bucket.rnodes): rnode_age = current_time - rnode.bucket_insertion_ts if rtt < rnode.rtt * (1 - (rnode_age / 7200)): # A rnode can only be replaced when the candidate node's RTT # is shorter by a factor. Over time, this factor # decreases. For instance, when rnode has been in the bucket # for 30 mins (1800 secs), a candidate's RTT must be at most # 50% of the rnode's RTT (ie. two times faster). After two # hours, a rnode cannot be replaced becouse of better RTT. # print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % ( rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts) rnode_to_be_replaced = rnode break if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) # No need to update table self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return # Get the worst node in replacement bucket and see whether # it's bad enough to be replaced by node_ worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: # This node is better candidate than worst_rnode r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) return
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = FILL_BUCKETS elif self._maintenance_mode == FILL_BUCKETS: if self._num_pending_filling_lookups: self._num_pending_filling_lookups -= 1 maintenance_lookup_target = identifier.RandomId() else: self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): # We try maintenance tasks till one of them actually does work # or we have tried them all (whatever happens first) We loop # in range because I'm going to modify self._maintenance_tasks task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] # This task did do some work. We are done here! break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): starting_index = self._next_stale_maintenance_index result = None while not result: # Find a non-empty bucket sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: # No node to be pinged in the whole table. break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: # Extra ping when bootstrapping num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ return def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: # Bootstrap nodes don't have id return message.OutgoingFindNodeQuery(node_, self.my_node.id, self.my_node.id, None) if random.choice((False, True)): # 50% chance to send find_node with my id as target return message.OutgoingFindNodeQuery(node_, self.my_node.id, self.my_node.id, None) # 50% chance to send a find_node to fill up a non-full bucket target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return message.OutgoingFindNodeQuery(node_, self.my_node.id, target, None) else: # Every bucket is full. We send a ping instead. return message.OutgoingPingQuery(node_, self.my_node.id) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket: queue it self._query_received_queue.add(node_, log_distance) return # No room in the main routing table # Add to replacement table (if the bucket is not full) worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) return def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) return # The node is not in main rnode = r_bucket.get_rnode(node_) if rnode: # node in replacement table # let's see whether there is room in the main self._update_rnode_on_response_received(rnode, rtt) #TODO: leave this for the maintenance task if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return # The node is nowhere # Add to main table (if the bucket is not full) #TODO: check whether in replacement_mode if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return # The main bucket is full # Let's see whether this node's latency is good current_time = time.time() rnode_to_be_replaced = None m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True) rnode = m_bucket.rnodes[0] if rtt < rnode.rtt: # Replace rnode (the node whose RTT is highest in the bucket) if # the candidate's RTT is lower regardless of rhe rnode's age. rnode_to_be_replaced = rnode if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) # No need to update table self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return # Get the worst node in replacement bucket and see whether # it's bad enough to be replaced by node_ worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: # This node is better candidate than worst_rnode r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) return def on_error_received(self, node_addr): pass def on_timeout(self, node_): if not node_.id: return [] # This is a bootstrap node (just addr, no id) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return [] # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: kick it out self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._replacement_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: # Replace worst node in replacement table r_bucket.remove(worst_rnode) r_bucket.add(rnode) # Node is not in main table rnode = r_bucket.get_rnode(node_) if rnode: # Node in replacement table: just update rnode self._update_rnode_on_timeout(rnode) return [] def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.rtt = rtt current_time = time.time() #rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = \ rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes, msg_f): self.my_node = my_node self.bootstrapper = bootstrap.OverlayBootstrapper(my_node.id, bootstrap_nodes, msg_f) self.msg_f = msg_f self.table = RoutingTable(my_node, NODES_PER_BUCKET) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS def _get_maintenance_lookup(self, lookup_target=None, nodes=[]): if not lookup_target: lookup_target = identifier.RandomId() if not nodes: log_distance = lookup_target.log_distance(self.my_node.id) nodes = self.get_closest_rnodes(log_distance, 0, True) return lookup_target, nodes def do_maintenance(self): queries_to_send = [] maintenance_lookup = None maintenance_delay = 0 if self._maintenance_mode == BOOTSTRAP_MODE: (queries_to_send, maintenance_lookup, bootstrap_delay) = self.bootstrapper.do_bootstrap( self.table.num_rnodes) if bootstrap_delay: maintenance_delay = bootstrap_delay else: self._maintenance_mode = FILL_BUCKETS elif self._maintenance_mode == FILL_BUCKETS: if self._num_pending_filling_lookups: self._num_pending_filling_lookups -= 1 maintenance_lookup = self._get_maintenance_lookup() else: self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): # We try maintenance tasks till one of them actually does work # or we have tried them all (whatever happens first) We loop # in range because I'm going to modify self._maintenance_tasks task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send.append(self._get_maintenance_query(node_)) # This task did do some work. We are done here! break if self.table.num_rnodes < MIN_RNODES: # Ping more found nodes when routing table has few nodes node_ = self._ping_a_found_node() if node_: queries_to_send.append(self._get_maintenance_query( node_, do_fill_up=True)) if not maintenance_delay: maintenance_delay = _MAINTENANCE_DELAY[self._maintenance_mode] return (maintenance_delay, queries_to_send, maintenance_lookup) def _ping_a_staled_rnode(self): starting_index = self._next_stale_maintenance_index result = None while not result: # Find a non-empty bucket sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: # No node to be pinged in the whole table. break return result def _ping_a_found_node(self): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_, do_fill_up=False): ''' if not node_.id: # Bootstrap nodes don't have id return message.OutgoingFindNodeQuery(node_, self.my_node.id, self.my_node.id, None) ''' if do_fill_up or random.choice((False, True)): # 50% chance to send a find_node to fill up a non-full bucket target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) msg = self.msg_f.outgoing_find_node_query(node_, target, None) else: # Every bucket is full. We send a ping instead. msg = self.msg_f.outgoing_ping_query(node_) else: # 50% chance to send find_node with my id as target msg = self.msg_f.outgoing_find_node_query(node_, self.my_node.id, None) return msg def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' if self.bootstrapper.is_bootstrap_node(node_): return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket: queue it self._query_received_queue.add(node_, log_distance) return # No room in the main routing table # Add to replacement table (if the bucket is not full) worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) return def on_response_received(self, node_, rtt, nodes): if self.bootstrapper.is_bootstrap_node(node_): return if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) return # The node is not in main rnode = r_bucket.get_rnode(node_) if rnode: # node in replacement table # let's see whether there is room in the main self._update_rnode_on_response_received(rnode, rtt) #TODO: leave this for the maintenance task if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return # The node is nowhere # Add to main table (if the bucket is not full) #TODO: check whether in replacement_mode if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return # The main bucket is full # Let's see whether this node's latency is good current_time = time.time() rnode_to_be_replaced = None m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True) for rnode in m_bucket.rnodes: rnode_age = current_time - rnode.bucket_insertion_ts if rtt < rnode.rtt * (1 - (rnode_age / 7200)): # A rnode can only be replaced when the candidate node's RTT # is shorter by a factor. Over time, this factor # decreases. For instance, when rnode has been in the bucket # for 30 mins (1800 secs), a candidate's RTT must be at most # 25% of the rnode's RTT (ie. two times faster). After two # hours, a rnode cannot be replaced by this method. # print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % ( # rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts) rnode_to_be_replaced = rnode break if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) # No need to update table self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return # Get the worst node in replacement bucket and see whether # it's bad enough to be replaced by node_ worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: # This node is better candidate than worst_rnode r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) return def on_error_received(self, node_addr): if self.bootstrapper.is_bootstrap_node(node_): return return def on_timeout(self, node_): if self.bootstrapper.is_bootstrap_node(node_): return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return [] # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: kick it out self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._replacement_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: # Replace worst node in replacement table r_bucket.remove(worst_rnode) r_bucket.add(rnode) # Node is not in main table rnode = r_bucket.get_rnode(node_) if rnode: # Node in replacement table: just update rnode self._update_rnode_on_timeout(rnode) return [] def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.rtt = rtt current_time = time.time() #rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = \ rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far