class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) # maintenance variables self._maintenance_mode = BOOTSTRAP_MODE self._pinged_q_rnodes = {} # questionable nodes which have been # recently pinged self._maintenance_tasks = [ self._refresh_stale_bucket, #self._ping_a_staled_rnode, # self._ping_a_query_received_node, # self._ping_a_found_node, ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = FIND_NODES_MODE return (10, [], maintenance_lookup_target) else: maintenance_lookup_target = self._refresh_stale_bucket() return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _refresh_stale_bucket(self): maintenance_lookup_target = None current_time = time.time() for i in xrange(self.table.lowest_index, NUM_BUCKETS): sbucket = self.table.get_sbucket(i) m_bucket = sbucket.main if not m_bucket: continue inactivity_time = current_time - m_bucket.last_changed_ts if inactivity_time > REFRESH_PERIOD: # print time.time(), '>>>>>>>>>>>>>>> refreshing bucket %d after %f secs' % ( # i, inactivity_time) maintenance_lookup_target = self.my_node.id.generate_close_id( i) m_bucket.last_changed_ts = current_time return maintenance_lookup_target self._maintenance_mode = NORMAL_MODE return None def _get_maintenance_query(self, node_): return Query(self.ping_msg, node_) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' if self._maintenance_mode != NORMAL_MODE: return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket. Just add the new node. rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_query_received(rnode) return # No room in the main routing table # Check whether there is a bad node to be replaced. bad_rnode = self._pop_bad_rnode(m_bucket) if bad_rnode: # We have a bad node in the bucket. Replace it with the new node. rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self._update_rnode_on_query_received(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 0 return # No bad nodes. Check for questionable nodes q_rnodes = self._get_questionable_rnodes(m_bucket) queries_to_send = [] # if q_rnodes: # print time.time(), '-----pinging questionable nodes in', # print log_distance # print q_rnodes for q_rnode in q_rnodes: # Ping questinable nodes to check whether they are still alive. # (0 timeouts so far, candidate node) c_rnode = node_.get_rnode(log_distance) self._update_rnode_on_query_received(c_rnode) self._pinged_q_rnodes[q_rnode] = [0, c_rnode] queries_to_send.append(Query(self.ping_msg, q_rnode)) return queries_to_send def on_response_received(self, node_, rtt, nodes): log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() if node_ in self._pinged_q_rnodes: # This node is questionable. This response proves that it is # alive. Remove it from the questionable dict. del self._pinged_q_rnodes[node_] return # The node is not in main if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() return # The main bucket is full # if there is a bad node inside the bucket, # replace it with the sending node_ bad_rnode = self._pop_bad_rnode(m_bucket) if bad_rnode: rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() self.table.update_lowest_index(log_distance) self.table.num_rnodes += 0 return # There are no bad nodes. Ping questionable nodes (if any) q_rnodes = self._get_questionable_rnodes(m_bucket) queries_to_send = [] for q_rnode in q_rnodes: # (0 timeouts so far, candidate node) c_rnode = node_.get_rnode(log_distance) self._update_rnode_on_response_received(c_rnode, rtt) self._pinged_q_rnodes[q_rnode] = [0, c_rnode] queries_to_send.append(Query(self.ping_msg, q_rnode)) return queries_to_send def _pop_bad_rnode(self, mbucket): for rnode in mbucket.rnodes: if rnode.timeouts_in_a_row() >= 2: mbucket.remove(rnode) return rnode def _get_questionable_rnodes(self, m_bucket): q_rnodes = [] for rnode in m_bucket.rnodes: inactivity_time = time.time() - rnode.last_seen if inactivity_time > REFRESH_PERIOD: q_rnodes.append(rnode) if rnode.num_responses == 0: q_rnodes.append(rnode) return q_rnodes def on_error_received(self, node_): pass def on_timeout(self, node_): if not node_.id: return # This is a bootstrap node (just addr, no id) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if not rnode: # This node is not in the table. Nothing to do here return # The node is in the table. Update it self._update_rnode_on_timeout(rnode) t_strikes, c_rnode = self._pinged_q_rnodes.get(node_, (None, None)) if t_strikes is None: # The node is not being checked by a "questinable ping". return elif t_strikes == 0: # This is the first timeout self._pinged_q_rnodes[node_] = (1, c_rnode) # Let's give it another chance return [Query(self.ping_msg, rnode)] elif t_strikes == 1: # Second timeout. You're a bad node, replace if possible # check if the candidate node is in the routing table log_distance = self.my_node.log_distance(c_rnode) m_bucket = self.table.get_sbucket(log_distance).main c_rnode_in_table = m_bucket.get_rnode(c_rnode) if c_rnode_in_table: print 'questionable node replaced' # replace m_bucket.remove(rnode) m_bucket.add(c_rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 0 def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.rtt = rtt current_time = time.time() #rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = rnode.last_action_ts < (current_time - QUARANTINE_PERIOD) rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery(my_node.id, my_node.id) self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except StopIteration: maintenance_lookup_target = self.my_node.id self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): starting_index = self._next_stale_maintenance_index result = None while not result: sbucket = self.table.get_sbucket(self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = (self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: return Query(self.find_closest_msg, node_) elif random.choice((False, True)): return Query(self.find_closest_msg, node_) target_log_distance = self.table.find_next_bucket_with_room_index(node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return Query(message.OutgoingFindNodeQuery(self.my_node.id, target), node_) else: return Query(self.ping_msg, node_) def on_query_received(self, node_): log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_query_received(rnode) return if m_bucket.there_is_room(): self._query_received_queue.add(node_, log_distance) return worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_response_received(rnode, rtt) return rnode = r_bucket.get_rnode(node_) if rnode: self._update_rnode_on_response_received(rnode, rtt) if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return current_time = time.time() rnode_to_be_replaced = None for rnode in reversed(m_bucket.rnodes): rnode_age = current_time - rnode.bucket_insertion_ts if rtt < rnode.rtt * (1 - rnode_age / 7200): rnode_to_be_replaced = rnode break if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) def on_error_received(self, node_addr): pass def on_timeout(self, node_): if not node_.id: return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._replacement_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: r_bucket.remove(worst_rnode) r_bucket.add(rnode) rnode = r_bucket.get_rnode(node_) if rnode: self._update_rnode_on_timeout(rnode) def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): rnode.rtt = rtt current_time = time.time() if rnode.in_quarantine: rnode.in_quarantine = rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [ self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): # We try maintenance tasks till one of them actually does work # or we have tried them all (whatever happens first) We loop # in range because I'm going to modify self._maintenance_tasks task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] # This task did do some work. We are done here! break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): # Don't have self._next_stale_maintenance_index lower than # lowest_bucket starting_index = self._next_stale_maintenance_index result = None while not result: # Find a non-empty bucket sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: # No node to be pinged in the whole table. break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: # Extra ping when bootstrapping num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ return def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: # Bootstrap nodes don't have id return Query(self.find_closest_msg, node_) if random.choice((False, True)): # 50% chance to send find_node with my id as target return Query(self.find_closest_msg, node_) # 50% chance to send a find_node to fill up a non-full bucket target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return Query( message.OutgoingFindNodeQuery(self.my_node.id, target), node_) else: # Every bucket is full. We send a ping instead. return Query(self.ping_msg, node_) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket: queue it self._query_received_queue.add(node_, log_distance) return # No room in the main routing table # Add to replacement table (if the bucket is not full) worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) return def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) return # The node is not in main rnode = r_bucket.get_rnode(node_) if rnode: # node in replacement table # let's see whether there is room in the main self._update_rnode_on_response_received(rnode, rtt) #TODO: leave this for the maintenance task if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return # The node is nowhere # Add to main table (if the bucket is not full) #TODO: check whether in replacement_mode if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return # The main bucket is full # Get the worst node in replacement bucket and see whether # it's bad enough to be replaced by node_ worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: # This node is better candidate than worst_rnode r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) return def on_error_received(self, node_): pass def on_timeout(self, node_): if not node_.id: return # This is a bootstrap node (just addr, no id) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: kick it out self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._replacement_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: # Replace worst node in replacement table r_bucket.remove(worst_rnode) r_bucket.add(rnode) # Node is not in main table rnode = r_bucket.get_rnode(node_) if rnode: # Node in replacement table: just update rnode self._update_rnode_on_timeout(rnode) def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.rtt = rtt current_time = time.time() #rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = \ rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): # We try maintenance tasks till one of them actually does work # or we have tried them all (whatever happens first) We loop # in range because I'm going to modify self._maintenance_tasks task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] # This task did do some work. We are done here! break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): # Don't have self._next_stale_maintenance_index lower than # lowest_bucket starting_index = self._next_stale_maintenance_index result = None while not result: # Find a non-empty bucket sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: # No node to be pinged in the whole table. break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: # Extra ping when bootstrapping num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ return def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: # Bootstrap nodes don't have id return Query(self.find_closest_msg, node_) if random.choice((False, True)): # 50% chance to send find_node with my id as target return Query(self.find_closest_msg, node_) # 50% chance to send a find_node to fill up a non-full bucket target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return Query( message.OutgoingFindNodeQuery(self.my_node.id, target), node_) else: # Every bucket is full. We send a ping instead. return Query(self.ping_msg, node_) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket: queue it self._query_received_queue.add(node_) return # No room in the main routing table # Add to replacement table (if the bucket is not full) worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) return def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) return # The node is not in main rnode = r_bucket.get_rnode(node_) if rnode: # node in replacement table # let's see whether there is room in the main self._update_rnode_on_response_received(rnode, rtt) #TODO: leave this for the maintenance task if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return # The node is nowhere # Add to main table (if the bucket is not full) #TODO: check whether in replacement_mode if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return # The main bucket is full # Let's see whether this node's latency is good current_time = time.time() rnode_to_be_replaced = None for rnode in reversed(m_bucket.rnodes): rnode_age = current_time - rnode.bucket_insertion_ts if rtt < rnode.rtt * (1 - (rnode_age / 7200)): # A rnode can only be replaced when the candidate node's RTT # is (at most) 50% of the rnode's. Over time, this factor # decreases. For instance, when rnode has been in the bucket # for 30 mins (1800 secs), a candidate's RTT must be at most # 25% of the rnode's RTT (ie. four times faster). After an # hour, a rnode cannot be replaced becouse of better RTT. print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % ( rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts) rnode_to_be_replaced = rnode break if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) # No need to update table self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return # Get the worst node in replacement bucket and see whether # it's bad enough to be replaced by node_ worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: # This node is better candidate than worst_rnode r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) return def on_error_received(self, node_): pass def on_timeout(self, node_): if not node_.id: return # This is a bootstrap node (just addr, no id) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: kick it out self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._query_received_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: # Replace worst node in replacement table r_bucket.remove(worst_rnode) r_bucket.add(rnode) # Node is not in main table rnode = r_bucket.get_rnode(node_) if rnode: # Node in replacement table: just update rnode self._update_rnode_on_timeout(rnode) def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.last_events.append((current_time, node.QUERY)) rnode.last_events[:rnode.max_last_events] rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.rtt = rtt current_time = time.time() #rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = \ rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.last_events.append((time.time(), node.RESPONSE)) rnode.last_events[:rnode.max_last_events] rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.last_events.append((time.time(), node.TIMEOUT)) rnode.last_events[:rnode.max_last_events] def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [ self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except StopIteration: maintenance_lookup_target = self.my_node.id self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): starting_index = self._next_stale_maintenance_index result = None while not result: sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: return Query(self.find_closest_msg, node_) elif random.choice((False, True)): return Query(self.find_closest_msg, node_) target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return Query( message.OutgoingFindNodeQuery(self.my_node.id, target), node_) else: return Query(self.ping_msg, node_) def on_query_received(self, node_): log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_query_received(rnode) return if m_bucket.there_is_room(): self._query_received_queue.add(node_, log_distance) return worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_response_received(rnode, rtt) return rnode = r_bucket.get_rnode(node_) if rnode: self._update_rnode_on_response_received(rnode, rtt) if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return current_time = time.time() rnode_to_be_replaced = None for rnode in reversed(m_bucket.rnodes): rnode_age = current_time - rnode.bucket_insertion_ts if rtt < rnode.rtt * (1 - rnode_age / 7200): rnode_to_be_replaced = rnode break if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) def on_error_received(self, node_addr): pass def on_timeout(self, node_): if not node_.id: return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except IndexError: return m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: self._update_rnode_on_timeout(rnode) m_bucket.remove(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes -= 1 for r_rnode in r_bucket.sorted_by_rtt(): self._replacement_queue.add(r_rnode) if r_bucket.there_is_room(): r_bucket.add(rnode) else: worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode: r_bucket.remove(worst_rnode) r_bucket.add(rnode) rnode = r_bucket.get_rnode(node_) if rnode: self._update_rnode_on_timeout(rnode) def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): rnode.rtt = rtt current_time = time.time() if rnode.in_quarantine: rnode.in_quarantine = rnode.last_action_ts < current_time - QUARANTINE_PERIOD rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) # maintenance variables self._next_stale_maintenance_index = 0 self._maintenance_mode = BOOTSTRAP_MODE self._replacement_queue = _ReplacementQueue(self.table) self._query_received_queue = _QueryReceivedQueue(self.table) self._found_nodes_queue = _FoundNodesQueue(self.table) self._maintenance_tasks = [self._ping_a_staled_rnode, self._ping_a_query_received_node, self._ping_a_found_node, self._ping_a_replacement_node, ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = NORMAL_MODE elif self._maintenance_mode == NORMAL_MODE: for _ in range(len(self._maintenance_tasks)): # We try maintenance tasks till one of them actually does work # or we have tried them all (whatever happens first) We loop # in range because I'm going to modify self._maintenance_tasks task = self._maintenance_tasks.pop(0) self._maintenance_tasks.append(task) node_ = task() if node_: queries_to_send = [self._get_maintenance_query(node_)] # This task did do some work. We are done here! break return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _ping_a_staled_rnode(self): # Don't have self._next_stale_maintenance_index lower than # lowest_bucket starting_index = self._next_stale_maintenance_index result = None while not result: # Find a non-empty bucket sbucket = self.table.get_sbucket( self._next_stale_maintenance_index) m_bucket = sbucket.main self._next_stale_maintenance_index = ( self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1) if m_bucket: rnode = m_bucket.get_stalest_rnode() if time.time() > rnode.last_seen + QUARANTINE_PERIOD: result = rnode if self._next_stale_maintenance_index == starting_index: # No node to be pinged in the whole table. break return result def _ping_a_found_node(self): num_pings = 1 if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP: # Extra ping when bootstrapping num_pings += 1 for _ in range(num_pings): node_ = self._found_nodes_queue.pop(0) if node_: logger.debug('pinging node found: %r', node_) return node_ return def _ping_a_query_received_node(self): return self._query_received_queue.pop(0) def _ping_a_replacement_node(self): return self._replacement_queue.pop(0) def _get_maintenance_query(self, node_): if not node_.id: # Bootstrap nodes don't have id return Query(self.find_closest_msg, node_) if random.choice((False, True)): # 50% chance to send find_node with my id as target return Query(self.find_closest_msg, node_) # 50% chance to send a find_node to fill up a non-full bucket target_log_distance = self.table.find_next_bucket_with_room_index( node_=node_) if target_log_distance: target = self.my_node.id.generate_close_id(target_log_distance) return Query( message.OutgoingFindNodeQuery(self.my_node.id, target), node_) else: # Every bucket is full. We send a ping instead. return Query(self.ping_msg, node_) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket: queue it self._query_received_queue.add(node_, log_distance) return # No room in the main routing table # Add to replacement table (if the bucket is not full) worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_query_received(rnode) return def on_response_received(self, node_, rtt, nodes): if nodes: logger.debug('nodes found: %r', nodes) self._found_nodes_queue.add(nodes) logger.debug('on response received %f', rtt) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main r_bucket = sbucket.replacement rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) return # The node is not in main rnode = r_bucket.get_rnode(node_) if rnode: # node in replacement table # let's see whether there is room in the main self._update_rnode_on_response_received(rnode, rtt) #TODO: leave this for the maintenance task if m_bucket.there_is_room(): m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) r_bucket.remove(rnode) return # The node is nowhere # Add to main table (if the bucket is not full) #TODO: check whether in replacement_mode if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) return # The main bucket is full # Let's see whether this node's latency is good current_time = time.time() rnode_to_be_replaced = None for rnode in reversed(m_bucket.rnodes): rnode_age = current_time - rnode.bucket_insertion_ts if rtt < rnode.rtt * (1 - (rnode_age / 7200)): # A rnode can only be replaced when the candidate node's RTT # is shorter by a factor. Over time, this factor # decreases. For instance, when rnode has been in the bucket # for 30 mins (1800 secs), a candidate's RTT must be at most # 50% of the rnode's RTT (ie. two times faster). After two # hours, a rnode cannot be replaced becouse of better RTT. # print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % ( rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts) rnode_to_be_replaced = rnode break if rnode_to_be_replaced: m_bucket.remove(rnode_to_be_replaced) rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) # No need to update table self.table.num_rnodes += 0 self._update_rnode_on_response_received(rnode, rtt) return # Get the worst node in replacement bucket and see whether # it's bad enough to be replaced by node_ worst_rnode = self._worst_rnode(r_bucket.rnodes) if worst_rnode \ and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS: # This node is better candidate than worst_rnode r_bucket.remove(worst_rnode) rnode = node_.get_rnode(log_distance) r_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) return
class RoutingManager(object): def __init__(self, my_node, bootstrap_nodes): self.my_node = my_node #Copy the bootstrap list self.bootstrap_nodes = iter(bootstrap_nodes) self.table = RoutingTable(my_node, NODES_PER_BUCKET) self.ping_msg = message.OutgoingPingQuery(my_node.id) self.find_closest_msg = message.OutgoingFindNodeQuery( my_node.id, my_node.id) # maintenance variables self._maintenance_mode = BOOTSTRAP_MODE self._pinged_q_rnodes = {} # questionable nodes which have been # recently pinged self._maintenance_tasks = [self._refresh_stale_bucket, #self._ping_a_staled_rnode, # self._ping_a_query_received_node, # self._ping_a_found_node, ] def do_maintenance(self): queries_to_send = [] maintenance_lookup_target = None if self._maintenance_mode == BOOTSTRAP_MODE: try: node_ = self.bootstrap_nodes.next() queries_to_send = [self._get_maintenance_query(node_)] except (StopIteration): maintenance_lookup_target = self.my_node.id self._maintenance_mode = FIND_NODES_MODE return (10, [], maintenance_lookup_target) else: maintenance_lookup_target = self._refresh_stale_bucket() return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target) def _refresh_stale_bucket(self): maintenance_lookup_target = None current_time = time.time() for i in xrange(self.table.lowest_index, NUM_BUCKETS): sbucket = self.table.get_sbucket(i) m_bucket = sbucket.main if not m_bucket: continue inactivity_time = current_time - m_bucket.last_changed_ts if inactivity_time > REFRESH_PERIOD: # print time.time(), '>>>>>>>>>>>>>>> refreshing bucket %d after %f secs' % ( # i, inactivity_time) maintenance_lookup_target = self.my_node.id.generate_close_id( i) m_bucket.last_changed_ts = current_time return maintenance_lookup_target self._maintenance_mode = NORMAL_MODE return None def _get_maintenance_query(self, node_): return Query(self.ping_msg, node_) def on_query_received(self, node_): ''' Return None when nothing to do Return a list of queries when queries need to be sent (the queries will be sent out by the caller) ''' if self._maintenance_mode != NORMAL_MODE: return log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a query from myself. Just ignore it. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: inform rnode self._update_rnode_on_query_received(rnode) return # node is not in the routing table if m_bucket.there_is_room(): # There is room in the bucket. Just add the new node. rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_query_received(rnode) return # No room in the main routing table # Check whether there is a bad node to be replaced. bad_rnode = self._pop_bad_rnode(m_bucket) if bad_rnode: # We have a bad node in the bucket. Replace it with the new node. rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self._update_rnode_on_query_received(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 0 return # No bad nodes. Check for questionable nodes q_rnodes = self._get_questionable_rnodes(m_bucket) queries_to_send = [] # if q_rnodes: # print time.time(), '-----pinging questionable nodes in', # print log_distance # print q_rnodes for q_rnode in q_rnodes: # Ping questinable nodes to check whether they are still alive. # (0 timeouts so far, candidate node) c_rnode = node_.get_rnode(log_distance) self._update_rnode_on_query_received(c_rnode) self._pinged_q_rnodes[q_rnode] = [0, c_rnode] queries_to_send.append(Query(self.ping_msg, q_rnode)) return queries_to_send def on_response_received(self, node_, rtt, nodes): log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except(IndexError): return # Got a response from myself. Just ignore it. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if rnode: # node in routing table: update self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() if node_ in self._pinged_q_rnodes: # This node is questionable. This response proves that it is # alive. Remove it from the questionable dict. del self._pinged_q_rnodes[node_] return # The node is not in main if m_bucket.there_is_room(): rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 1 self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() return # The main bucket is full # if there is a bad node inside the bucket, # replace it with the sending node_ bad_rnode = self._pop_bad_rnode(m_bucket) if bad_rnode: rnode = node_.get_rnode(log_distance) m_bucket.add(rnode) self._update_rnode_on_response_received(rnode, rtt) if self._maintenance_mode == NORMAL_MODE: m_bucket.last_changed_ts = time.time() self.table.update_lowest_index(log_distance) self.table.num_rnodes += 0 return # There are no bad nodes. Ping questionable nodes (if any) q_rnodes = self._get_questionable_rnodes(m_bucket) queries_to_send = [] for q_rnode in q_rnodes: # (0 timeouts so far, candidate node) c_rnode = node_.get_rnode(log_distance) self._update_rnode_on_response_received(c_rnode, rtt) self._pinged_q_rnodes[q_rnode] = [0, c_rnode] queries_to_send.append(Query(self.ping_msg, q_rnode)) return queries_to_send def _pop_bad_rnode(self, mbucket): for rnode in mbucket.rnodes: if rnode.timeouts_in_a_row() >= 2: mbucket.remove(rnode) return rnode def _get_questionable_rnodes(self, m_bucket): q_rnodes = [] for rnode in m_bucket.rnodes: inactivity_time = time.time() - rnode.last_seen if inactivity_time > REFRESH_PERIOD: q_rnodes.append(rnode) if rnode.num_responses == 0: q_rnodes.append(rnode) return q_rnodes def on_error_received(self, node_): pass def on_timeout(self, node_): if not node_.id: return # This is a bootstrap node (just addr, no id) log_distance = self.my_node.log_distance(node_) try: sbucket = self.table.get_sbucket(log_distance) except (IndexError): return # Got a timeout from myself, WTF? Just ignore. m_bucket = sbucket.main rnode = m_bucket.get_rnode(node_) if not rnode: # This node is not in the table. Nothing to do here return # The node is in the table. Update it self._update_rnode_on_timeout(rnode) t_strikes, c_rnode = self._pinged_q_rnodes.get(node_, (None, None)) if t_strikes is None: # The node is not being checked by a "questinable ping". return elif t_strikes == 0: # This is the first timeout self._pinged_q_rnodes[node_] = (1, c_rnode) # Let's give it another chance return [Query(self.ping_msg, rnode)] elif t_strikes == 1: # Second timeout. You're a bad node, replace if possible # check if the candidate node is in the routing table log_distance = self.my_node.log_distance(c_rnode) m_bucket = self.table.get_sbucket(log_distance).main c_rnode_in_table = m_bucket.get_rnode(c_rnode) if c_rnode_in_table: print 'questionable node replaced' # replace m_bucket.remove(rnode) m_bucket.add(c_rnode) self.table.update_lowest_index(log_distance) self.table.num_rnodes += 0 def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself): if not num_nodes: num_nodes = NODES_PER_BUCKET[log_distance] return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself) def get_main_rnodes(self): return self.table.get_main_rnodes() def print_stats(self): self.table.print_stats() def _update_rnode_on_query_received(self, rnode): """Register a query from node. You should call this method when receiving a query from this node. """ current_time = time.time() rnode.last_action_ts = time.time() rnode.msgs_since_timeout += 1 rnode.num_queries += 1 rnode.add_event(current_time, node.QUERY) rnode.last_seen = current_time def _update_rnode_on_response_received(self, rnode, rtt): """Register a reply from rnode. You should call this method when receiving a response from this rnode. """ rnode.rtt = rtt current_time = time.time() #rnode._reset_refresh_task() if rnode.in_quarantine: rnode.in_quarantine = rnode.last_action_ts < ( current_time - QUARANTINE_PERIOD) rnode.last_action_ts = current_time rnode.num_responses += 1 rnode.add_event(time.time(), node.RESPONSE) rnode.last_seen = current_time def _update_rnode_on_timeout(self, rnode): """Register a timeout for this rnode. You should call this method when getting a timeout for this node. """ rnode.last_action_ts = time.time() rnode.msgs_since_timeout = 0 rnode.num_timeouts += 1 rnode.add_event(time.time(), node.TIMEOUT) def _worst_rnode(self, rnodes): max_num_timeouts = -1 worst_rnode_so_far = None for rnode in rnodes: num_timeouots = rnode.timeouts_in_a_row() if num_timeouots >= max_num_timeouts: max_num_timeouts = num_timeouots worst_rnode_so_far = rnode return worst_rnode_so_far