class RoutingManager(object):
    
    def __init__(self, my_node, bootstrap_nodes):
        self.my_node = my_node
        #Copy the bootstrap list
        self.bootstrap_nodes = iter(bootstrap_nodes)
        
        self.table = RoutingTable(my_node, NODES_PER_BUCKET)
        # maintenance variables
        self._next_stale_maintenance_index = 0
        self._maintenance_mode = BOOTSTRAP_MODE
        self._replacement_queue = _ReplacementQueue(self.table)
        self._query_received_queue = _QueryReceivedQueue(self.table)
        self._found_nodes_queue = _FoundNodesQueue(self.table)
        self._maintenance_tasks = [self._ping_a_staled_rnode,
                                   self._ping_a_query_received_node,
                                   self._ping_a_found_node,
                                   self._ping_a_replacement_node,
                                   ]
        self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
        
    def do_maintenance(self):
        queries_to_send = []
        maintenance_lookup_target = None
        if self._maintenance_mode == BOOTSTRAP_MODE:
            try:
                node_ = self.bootstrap_nodes.next()
                queries_to_send = [self._get_maintenance_query(node_)]
            except (StopIteration):
                maintenance_lookup_target = self.my_node.id
                self._maintenance_mode = FILL_BUCKETS
        elif self._maintenance_mode == FILL_BUCKETS:
            if self._num_pending_filling_lookups:
                self._num_pending_filling_lookups -= 1
                maintenance_lookup_target = identifier.RandomId()
            else:
                self._maintenance_mode = NORMAL_MODE
        elif self._maintenance_mode == NORMAL_MODE:
            for _ in range(len(self._maintenance_tasks)):
                # We try maintenance tasks till one of them actually does work
                # or we have tried them all (whatever happens first) We loop
                # in range because I'm going to modify self._maintenance_tasks
                task = self._maintenance_tasks.pop(0)
                self._maintenance_tasks.append(task)
                node_ = task()
                if node_:
                    queries_to_send = [self._get_maintenance_query(node_)]
                    # This task did do some work. We are done here!
                    break
        
        return (_MAINTENANCE_DELAY[self._maintenance_mode],
                queries_to_send, maintenance_lookup_target)

    def _ping_a_staled_rnode(self):
        starting_index = self._next_stale_maintenance_index
        result = None
        while not result:
            # Find a non-empty bucket
            sbucket = self.table.get_sbucket(
                self._next_stale_maintenance_index)
            m_bucket = sbucket.main
            self._next_stale_maintenance_index = (
                self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
            if m_bucket:
                rnode = m_bucket.get_stalest_rnode()
                if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
                    result = rnode
            if self._next_stale_maintenance_index == starting_index:
                # No node to be pinged in the whole table.
                break
        return result

    def _ping_a_found_node(self):
        num_pings = 1
        if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP:
            # Extra ping when bootstrapping
            num_pings += 1
        for _ in range(num_pings):
            node_ = self._found_nodes_queue.pop(0)
            if node_:
                logger.debug('pinging node found: %r', node_)
                return node_
        return

    def _ping_a_query_received_node(self):
        return self._query_received_queue.pop(0)

    def _ping_a_replacement_node(self):
        return self._replacement_queue.pop(0)
                                  
    def _get_maintenance_query(self, node_):
        if not node_.id: 
            # Bootstrap nodes don't have id
            return message.OutgoingFindNodeQuery(node_,
                                                 self.my_node.id,
                                                 self.my_node.id, None)
        if random.choice((False, True)):
            # 50% chance to send find_node with my id as target
            return message.OutgoingFindNodeQuery(node_,
                                                 self.my_node.id,
                                                 self.my_node.id, None)

        # 50% chance to send a find_node to fill up a non-full bucket
        target_log_distance = self.table.find_next_bucket_with_room_index(
            node_=node_)
        if target_log_distance:
            target = self.my_node.id.generate_close_id(target_log_distance)
            return message.OutgoingFindNodeQuery(node_, self.my_node.id,
                                                 target, None)
        else:
            # Every bucket is full. We send a ping instead.
            return message.OutgoingPingQuery(node_, self.my_node.id)
        
    def on_query_received(self, node_):
        '''
        Return None when nothing to do
        Return a list of queries when queries need to be sent (the queries
        will be sent out by the caller)
        '''
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except(IndexError):
            return # Got a query from myself. Just ignore it.

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: inform rnode
            self._update_rnode_on_query_received(rnode)
            return
        
        # node is not in the routing table
        if m_bucket.there_is_room():
            # There is room in the bucket: queue it
            self._query_received_queue.add(node_, log_distance)
            return
        # No room in the main routing table
        # Add to replacement table (if the bucket is not full)
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode \
                and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_query_received(rnode)
        return
            
    def on_response_received(self, node_, rtt, nodes):
        if nodes:
            logger.debug('nodes found: %r', nodes)
        self._found_nodes_queue.add(nodes)

        logger.debug('on response received %f', rtt)
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except(IndexError):
            return # Got a response from myself. Just ignore it.
        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: update
            self._update_rnode_on_response_received(rnode, rtt)
            return
        # The node is not in main
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            # node in replacement table
            # let's see whether there is room in the main
            self._update_rnode_on_response_received(rnode, rtt)
            #TODO: leave this for the maintenance task
            if m_bucket.there_is_room():
                m_bucket.add(rnode)
                self.table.num_rnodes += 1
                self._update_rnode_on_response_received(rnode, rtt)
                r_bucket.remove(rnode)
            return
        # The node is nowhere
        # Add to main table (if the bucket is not full)
        #TODO: check whether in replacement_mode
        if m_bucket.there_is_room():
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.num_rnodes += 1
            self._update_rnode_on_response_received(rnode, rtt)
            return
        # The main bucket is full
        # Let's see whether this node's latency is good
        current_time = time.time()
        rnode_to_be_replaced = None
        m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True)
        rnode = m_bucket.rnodes[0]
        if rtt < rnode.rtt:
            # Replace rnode (the node whose RTT is highest in the bucket) if
            # the candidate's RTT is lower regardless of rhe rnode's age.
            rnode_to_be_replaced = rnode
        if rnode_to_be_replaced:
            m_bucket.remove(rnode_to_be_replaced)
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            # No need to update table
            self.table.num_rnodes += 0
            self._update_rnode_on_response_received(rnode, rtt)
            return
            
        # Get the worst node in replacement bucket and see whether
        # it's bad enough to be replaced by node_
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode \
                and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            # This node is better candidate than worst_rnode
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_response_received(rnode, rtt)
        return
        
    def on_error_received(self, node_addr):
        pass
    
    def on_timeout(self, node_):
        if not node_.id:
            return [] # This is a bootstrap node (just addr, no id)
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return [] # Got a timeout from myself, WTF? Just ignore.
        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: kick it out
            self._update_rnode_on_timeout(rnode)
            m_bucket.remove(rnode)
            self.table.num_rnodes -= 1

            for r_rnode in r_bucket.sorted_by_rtt():
                self._replacement_queue.add(r_rnode)
            if r_bucket.there_is_room():
                r_bucket.add(rnode)
            else:
                worst_rnode = self._worst_rnode(r_bucket.rnodes)
                if worst_rnode:
                    # Replace worst node in replacement table
                    r_bucket.remove(worst_rnode)
                    r_bucket.add(rnode)
        # Node is not in main table
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            # Node in replacement table: just update rnode
            self._update_rnode_on_timeout(rnode)
        return []
            
    def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
        if not num_nodes:
            num_nodes = NODES_PER_BUCKET[log_distance]
        return self.table.get_closest_rnodes(log_distance, num_nodes,
                                             exclude_myself)

    def get_main_rnodes(self):
        return self.table.get_main_rnodes()

    def print_stats(self):
        self.table.print_stats()

    def _update_rnode_on_query_received(self, rnode):
        """Register a query from node.

        You should call this method when receiving a query from this node.

        """
        current_time = time.time()
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout += 1
        rnode.num_queries += 1
        rnode.add_event(current_time, node.QUERY)
        rnode.last_seen = current_time

    def _update_rnode_on_response_received(self, rnode, rtt):
        """Register a reply from rnode.

        You should call this method when receiving a response from this rnode.

        """
        rnode.rtt = rtt
        current_time = time.time()
        #rnode._reset_refresh_task()
        if rnode.in_quarantine:
            rnode.in_quarantine = \
                rnode.last_action_ts < current_time - QUARANTINE_PERIOD
                
        rnode.last_action_ts = current_time
        rnode.num_responses += 1
        rnode.add_event(time.time(), node.RESPONSE)
        rnode.last_seen = current_time

    def _update_rnode_on_timeout(self, rnode):
        """Register a timeout for this rnode.

        You should call this method when getting a timeout for this node.

        """
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout = 0
        rnode.num_timeouts += 1
        rnode.add_event(time.time(), node.TIMEOUT)

    def _worst_rnode(self, rnodes):
        max_num_timeouts = -1
        worst_rnode_so_far = None
        for rnode in rnodes:
            num_timeouots = rnode.timeouts_in_a_row()
            if num_timeouots >= max_num_timeouts:
                max_num_timeouts = num_timeouots
                worst_rnode_so_far = rnode
        return worst_rnode_so_far
Beispiel #2
0
class RoutingManager(object):
    def __init__(self, my_node, bootstrap_nodes):
        self.my_node = my_node
        #Copy the bootstrap list
        self.bootstrap_nodes = iter(bootstrap_nodes)

        self.table = RoutingTable(my_node, NODES_PER_BUCKET)
        self.ping_msg = message.OutgoingPingQuery(my_node.id)
        self.find_closest_msg = message.OutgoingFindNodeQuery(
            my_node.id, my_node.id)

        # maintenance variables
        self._maintenance_mode = BOOTSTRAP_MODE
        self._pinged_q_rnodes = {}  # questionable nodes which have been
        # recently pinged
        self._maintenance_tasks = [
            self._refresh_stale_bucket,
            #self._ping_a_staled_rnode,
            # self._ping_a_query_received_node,
            # self._ping_a_found_node,
        ]

    def do_maintenance(self):
        queries_to_send = []
        maintenance_lookup_target = None
        if self._maintenance_mode == BOOTSTRAP_MODE:
            try:
                node_ = self.bootstrap_nodes.next()
                queries_to_send = [self._get_maintenance_query(node_)]
            except (StopIteration):
                maintenance_lookup_target = self.my_node.id
                self._maintenance_mode = FIND_NODES_MODE
                return (10, [], maintenance_lookup_target)
        else:
            maintenance_lookup_target = self._refresh_stale_bucket()

        return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send,
                maintenance_lookup_target)

    def _refresh_stale_bucket(self):
        maintenance_lookup_target = None
        current_time = time.time()
        for i in xrange(self.table.lowest_index, NUM_BUCKETS):
            sbucket = self.table.get_sbucket(i)
            m_bucket = sbucket.main
            if not m_bucket:
                continue
            inactivity_time = current_time - m_bucket.last_changed_ts
            if inactivity_time > REFRESH_PERIOD:
                #                print time.time(), '>>>>>>>>>>>>>>> refreshing bucket %d after %f secs' % (
                #                    i, inactivity_time)
                maintenance_lookup_target = self.my_node.id.generate_close_id(
                    i)
                m_bucket.last_changed_ts = current_time
                return maintenance_lookup_target
        self._maintenance_mode = NORMAL_MODE
        return None

    def _get_maintenance_query(self, node_):
        return Query(self.ping_msg, node_)

    def on_query_received(self, node_):
        '''
        Return None when nothing to do
        Return a list of queries when queries need to be sent (the queries
        will be sent out by the caller)
        '''
        if self._maintenance_mode != NORMAL_MODE:
            return
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return  # Got a query from myself. Just ignore it.

        m_bucket = sbucket.main
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: inform rnode
            self._update_rnode_on_query_received(rnode)
            return

        # node is not in the routing table
        if m_bucket.there_is_room():
            # There is room in the bucket. Just add the new node.
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes += 1
            self._update_rnode_on_query_received(rnode)
            return
        # No room in the main routing table
        # Check whether there is a bad node to be replaced.
        bad_rnode = self._pop_bad_rnode(m_bucket)
        if bad_rnode:
            # We have a bad node in the bucket. Replace it with the new node.
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self._update_rnode_on_query_received(rnode)
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes += 0
            return

        # No bad nodes. Check for questionable nodes
        q_rnodes = self._get_questionable_rnodes(m_bucket)
        queries_to_send = []
        #        if q_rnodes:
        #            print time.time(), '-----pinging questionable nodes in',
        #            print log_distance
        #            print q_rnodes
        for q_rnode in q_rnodes:
            # Ping questinable nodes to check whether they are still alive.
            # (0 timeouts so far, candidate node)
            c_rnode = node_.get_rnode(log_distance)
            self._update_rnode_on_query_received(c_rnode)
            self._pinged_q_rnodes[q_rnode] = [0, c_rnode]
            queries_to_send.append(Query(self.ping_msg, q_rnode))
        return queries_to_send

    def on_response_received(self, node_, rtt, nodes):
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return  # Got a response from myself. Just ignore it.
        m_bucket = sbucket.main
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: update
            self._update_rnode_on_response_received(rnode, rtt)
            if self._maintenance_mode == NORMAL_MODE:
                m_bucket.last_changed_ts = time.time()
            if node_ in self._pinged_q_rnodes:
                # This node is questionable. This response proves that it is
                # alive. Remove it from the questionable dict.
                del self._pinged_q_rnodes[node_]
            return

        # The node is not in main
        if m_bucket.there_is_room():
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes += 1
            self._update_rnode_on_response_received(rnode, rtt)
            if self._maintenance_mode == NORMAL_MODE:
                m_bucket.last_changed_ts = time.time()
            return
        # The main bucket is full

        # if there is a bad node inside the bucket,
        # replace it with the sending node_
        bad_rnode = self._pop_bad_rnode(m_bucket)
        if bad_rnode:
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self._update_rnode_on_response_received(rnode, rtt)
            if self._maintenance_mode == NORMAL_MODE:
                m_bucket.last_changed_ts = time.time()
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes += 0
            return

        # There are no bad nodes. Ping questionable nodes (if any)
        q_rnodes = self._get_questionable_rnodes(m_bucket)
        queries_to_send = []
        for q_rnode in q_rnodes:
            # (0 timeouts so far, candidate node)
            c_rnode = node_.get_rnode(log_distance)
            self._update_rnode_on_response_received(c_rnode, rtt)
            self._pinged_q_rnodes[q_rnode] = [0, c_rnode]
            queries_to_send.append(Query(self.ping_msg, q_rnode))
        return queries_to_send

    def _pop_bad_rnode(self, mbucket):
        for rnode in mbucket.rnodes:
            if rnode.timeouts_in_a_row() >= 2:
                mbucket.remove(rnode)
                return rnode

    def _get_questionable_rnodes(self, m_bucket):
        q_rnodes = []
        for rnode in m_bucket.rnodes:
            inactivity_time = time.time() - rnode.last_seen
            if inactivity_time > REFRESH_PERIOD:
                q_rnodes.append(rnode)
            if rnode.num_responses == 0:
                q_rnodes.append(rnode)
        return q_rnodes

    def on_error_received(self, node_):
        pass

    def on_timeout(self, node_):
        if not node_.id:
            return  # This is a bootstrap node (just addr, no id)
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return  # Got a timeout from myself, WTF? Just ignore.
        m_bucket = sbucket.main
        rnode = m_bucket.get_rnode(node_)

        if not rnode:
            # This node is not in the table. Nothing to do here
            return

        # The node is in the table. Update it
        self._update_rnode_on_timeout(rnode)
        t_strikes, c_rnode = self._pinged_q_rnodes.get(node_, (None, None))
        if t_strikes is None:
            # The node is not being checked by a "questinable ping".
            return
        elif t_strikes == 0:
            # This is the first timeout
            self._pinged_q_rnodes[node_] = (1, c_rnode)
            # Let's give it another chance
            return [Query(self.ping_msg, rnode)]
        elif t_strikes == 1:
            # Second timeout. You're a bad node, replace if possible
            # check if the candidate node is in the routing table
            log_distance = self.my_node.log_distance(c_rnode)
            m_bucket = self.table.get_sbucket(log_distance).main
            c_rnode_in_table = m_bucket.get_rnode(c_rnode)
            if c_rnode_in_table:
                print 'questionable node replaced'
                # replace
                m_bucket.remove(rnode)
                m_bucket.add(c_rnode)
                self.table.update_lowest_index(log_distance)
                self.table.num_rnodes += 0

    def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
        if not num_nodes:
            num_nodes = NODES_PER_BUCKET[log_distance]
        return self.table.get_closest_rnodes(log_distance, num_nodes,
                                             exclude_myself)

    def get_main_rnodes(self):
        return self.table.get_main_rnodes()

    def print_stats(self):
        self.table.print_stats()

    def _update_rnode_on_query_received(self, rnode):
        """Register a query from node.

        You should call this method when receiving a query from this node.

        """
        current_time = time.time()
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout += 1
        rnode.num_queries += 1
        rnode.add_event(current_time, node.QUERY)
        rnode.last_seen = current_time

    def _update_rnode_on_response_received(self, rnode, rtt):
        """Register a reply from rnode.

        You should call this method when receiving a response from this rnode.

        """
        rnode.rtt = rtt
        current_time = time.time()
        #rnode._reset_refresh_task()
        if rnode.in_quarantine:
            rnode.in_quarantine = rnode.last_action_ts < (current_time -
                                                          QUARANTINE_PERIOD)
        rnode.last_action_ts = current_time
        rnode.num_responses += 1
        rnode.add_event(time.time(), node.RESPONSE)
        rnode.last_seen = current_time

    def _update_rnode_on_timeout(self, rnode):
        """Register a timeout for this rnode.

        You should call this method when getting a timeout for this node.

        """
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout = 0
        rnode.num_timeouts += 1
        rnode.add_event(time.time(), node.TIMEOUT)

    def _worst_rnode(self, rnodes):
        max_num_timeouts = -1
        worst_rnode_so_far = None
        for rnode in rnodes:
            num_timeouots = rnode.timeouts_in_a_row()
            if num_timeouots >= max_num_timeouts:
                max_num_timeouts = num_timeouots
                worst_rnode_so_far = rnode
        return worst_rnode_so_far
class RoutingManager(object):

    def __init__(self, my_node, bootstrap_nodes):
        self.my_node = my_node
        self.bootstrap_nodes = iter(bootstrap_nodes)
        self.table = RoutingTable(my_node, NODES_PER_BUCKET)
        self.ping_msg = message.OutgoingPingQuery(my_node.id)
        self.find_closest_msg = message.OutgoingFindNodeQuery(my_node.id, my_node.id)
        self._next_stale_maintenance_index = 0
        self._maintenance_mode = BOOTSTRAP_MODE
        self._replacement_queue = _ReplacementQueue(self.table)
        self._query_received_queue = _QueryReceivedQueue(self.table)
        self._found_nodes_queue = _FoundNodesQueue(self.table)
        self._maintenance_tasks = [self._ping_a_staled_rnode,
         self._ping_a_query_received_node,
         self._ping_a_found_node,
         self._ping_a_replacement_node]

    def do_maintenance(self):
        queries_to_send = []
        maintenance_lookup_target = None
        if self._maintenance_mode == BOOTSTRAP_MODE:
            try:
                node_ = self.bootstrap_nodes.next()
                queries_to_send = [self._get_maintenance_query(node_)]
            except StopIteration:
                maintenance_lookup_target = self.my_node.id
                self._maintenance_mode = NORMAL_MODE

        elif self._maintenance_mode == NORMAL_MODE:
            for _ in range(len(self._maintenance_tasks)):
                task = self._maintenance_tasks.pop(0)
                self._maintenance_tasks.append(task)
                node_ = task()
                if node_:
                    queries_to_send = [self._get_maintenance_query(node_)]
                    break

        return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target)

    def _ping_a_staled_rnode(self):
        starting_index = self._next_stale_maintenance_index
        result = None
        while not result:
            sbucket = self.table.get_sbucket(self._next_stale_maintenance_index)
            m_bucket = sbucket.main
            self._next_stale_maintenance_index = (self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
            if m_bucket:
                rnode = m_bucket.get_stalest_rnode()
                if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
                    result = rnode
            if self._next_stale_maintenance_index == starting_index:
                break

        return result

    def _ping_a_found_node(self):
        num_pings = 1
        if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP:
            num_pings += 1
        for _ in range(num_pings):
            node_ = self._found_nodes_queue.pop(0)
            if node_:
                logger.debug('pinging node found: %r', node_)
                return node_

    def _ping_a_query_received_node(self):
        return self._query_received_queue.pop(0)

    def _ping_a_replacement_node(self):
        return self._replacement_queue.pop(0)

    def _get_maintenance_query(self, node_):
        if not node_.id:
            return Query(self.find_closest_msg, node_)
        elif random.choice((False, True)):
            return Query(self.find_closest_msg, node_)
        target_log_distance = self.table.find_next_bucket_with_room_index(node_=node_)
        if target_log_distance:
            target = self.my_node.id.generate_close_id(target_log_distance)
            return Query(message.OutgoingFindNodeQuery(self.my_node.id, target), node_)
        else:
            return Query(self.ping_msg, node_)

    def on_query_received(self, node_):
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except IndexError:
            return

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_query_received(rnode)
            return
        if m_bucket.there_is_room():
            self._query_received_queue.add(node_, log_distance)
            return
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_query_received(rnode)

    def on_response_received(self, node_, rtt, nodes):
        if nodes:
            logger.debug('nodes found: %r', nodes)
        self._found_nodes_queue.add(nodes)
        logger.debug('on response received %f', rtt)
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except IndexError:
            return

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_response_received(rnode, rtt)
            return
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_response_received(rnode, rtt)
            if m_bucket.there_is_room():
                m_bucket.add(rnode)
                self.table.update_lowest_index(log_distance)
                self.table.num_rnodes += 1
                self._update_rnode_on_response_received(rnode, rtt)
                r_bucket.remove(rnode)
            return
        if m_bucket.there_is_room():
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes += 1
            self._update_rnode_on_response_received(rnode, rtt)
            return
        current_time = time.time()
        rnode_to_be_replaced = None
        for rnode in reversed(m_bucket.rnodes):
            rnode_age = current_time - rnode.bucket_insertion_ts
            if rtt < rnode.rtt * (1 - rnode_age / 7200):
                rnode_to_be_replaced = rnode
                break

        if rnode_to_be_replaced:
            m_bucket.remove(rnode_to_be_replaced)
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.num_rnodes += 0
            self._update_rnode_on_response_received(rnode, rtt)
            return
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_response_received(rnode, rtt)

    def on_error_received(self, node_addr):
        pass

    def on_timeout(self, node_):
        if not node_.id:
            return
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except IndexError:
            return

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_timeout(rnode)
            m_bucket.remove(rnode)
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes -= 1
            for r_rnode in r_bucket.sorted_by_rtt():
                self._replacement_queue.add(r_rnode)

            if r_bucket.there_is_room():
                r_bucket.add(rnode)
            else:
                worst_rnode = self._worst_rnode(r_bucket.rnodes)
                if worst_rnode:
                    r_bucket.remove(worst_rnode)
                    r_bucket.add(rnode)
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_timeout(rnode)

    def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
        if not num_nodes:
            num_nodes = NODES_PER_BUCKET[log_distance]
        return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself)

    def get_main_rnodes(self):
        return self.table.get_main_rnodes()

    def print_stats(self):
        self.table.print_stats()

    def _update_rnode_on_query_received(self, rnode):
        current_time = time.time()
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout += 1
        rnode.num_queries += 1
        rnode.add_event(current_time, node.QUERY)
        rnode.last_seen = current_time

    def _update_rnode_on_response_received(self, rnode, rtt):
        rnode.rtt = rtt
        current_time = time.time()
        if rnode.in_quarantine:
            rnode.in_quarantine = rnode.last_action_ts < current_time - QUARANTINE_PERIOD
        rnode.last_action_ts = current_time
        rnode.num_responses += 1
        rnode.add_event(time.time(), node.RESPONSE)
        rnode.last_seen = current_time

    def _update_rnode_on_timeout(self, rnode):
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout = 0
        rnode.num_timeouts += 1
        rnode.add_event(time.time(), node.TIMEOUT)

    def _worst_rnode(self, rnodes):
        max_num_timeouts = -1
        worst_rnode_so_far = None
        for rnode in rnodes:
            num_timeouots = rnode.timeouts_in_a_row()
            if num_timeouots >= max_num_timeouts:
                max_num_timeouts = num_timeouots
                worst_rnode_so_far = rnode

        return worst_rnode_so_far
Beispiel #4
0
class RoutingManager(object):
    
    def __init__(self, my_node, bootstrap_nodes):
        self.my_node = my_node
        #Copy the bootstrap list
        self.bootstrap_nodes = iter(bootstrap_nodes)
        
        self.table = RoutingTable(my_node, NODES_PER_BUCKET)
        # maintenance variables
        self._maintenance_mode = BOOTSTRAP_MODE
        self._pinged_q_rnodes = {} # questionable nodes which have been
        # recently pinged
        self._maintenance_tasks = [self._refresh_stale_bucket,
            #self._ping_a_staled_rnode,
            # self._ping_a_query_received_node,
            # self._ping_a_found_node,
                                   ]
        
    def do_maintenance(self):
        queries_to_send = []
        maintenance_lookup_target = None
        if self._maintenance_mode == BOOTSTRAP_MODE:
            try:
                node_ = self.bootstrap_nodes.next()
                queries_to_send = [self._get_maintenance_query(node_)]
            except (StopIteration):
                maintenance_lookup_target = self.my_node.id
                self._maintenance_mode = FIND_NODES_MODE
                return (10, [], maintenance_lookup_target)
        else:
            maintenance_lookup_target = self._refresh_stale_bucket()

#        print 'beb5', _MAINTENANCE_DELAY[self._maintenance_mode]
        return (_MAINTENANCE_DELAY[self._maintenance_mode],
                queries_to_send, maintenance_lookup_target)
    
    def _refresh_stale_bucket(self):
        maintenance_lookup_target = None
        current_time = time.time()
        for i in xrange(NUM_BUCKETS):
            sbucket = self.table.get_sbucket(i)
            m_bucket = sbucket.main
            if not m_bucket:
                continue
            inactivity_time = current_time - m_bucket.last_changed_ts
            if inactivity_time > REFRESH_PERIOD:
#                print time.time(), '>>>>>>>>>>>>>>> refreshing bucket %d after %f secs' % (
#                    i, inactivity_time)
                maintenance_lookup_target = self.my_node.id.generate_close_id(
                    i)
                m_bucket.last_changed_ts = current_time
                return maintenance_lookup_target
        self._maintenance_mode = NORMAL_MODE
        return None

    def _get_maintenance_query(self, node_):
        return message.OutgoingPingQuery(node_, self.my_node.id)
         
    def on_query_received(self, node_):
        '''
        Return None when nothing to do
        Return a list of queries when queries need to be sent (the queries
        will be sent out by the caller)
        '''
        if self._maintenance_mode != NORMAL_MODE:
            return
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except(IndexError):
            return # Got a query from myself. Just ignore it.

        m_bucket = sbucket.main
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: inform rnode
            self._update_rnode_on_query_received(rnode)
            return
        
        # node is not in the routing table
        if m_bucket.there_is_room():
            # There is room in the bucket. Just add the new node.
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.num_rnodes += 1
            self._update_rnode_on_query_received(rnode)
            return
        # No room in the main routing table
        # Check whether there is a bad node to be replaced.
        bad_rnode = self._pop_bad_rnode(m_bucket)
        if bad_rnode:
            # We have a bad node in the bucket. Replace it with the new node.
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self._update_rnode_on_query_received(rnode)
            self.table.num_rnodes += 0
            return

        # No bad nodes. Check for questionable nodes
        q_rnodes = self._get_questionable_rnodes(m_bucket)
        queries_to_send = []
        for q_rnode in q_rnodes:
            # Ping questinable nodes to check whether they are still alive.
            # (0 timeouts so far, candidate node)
            c_rnode = node_.get_rnode(log_distance)
            self._update_rnode_on_query_received(c_rnode)
            self._pinged_q_rnodes[q_rnode] = [0, c_rnode]
            queries_to_send.append(message.OutgoingPingQuery(node_, self.my_node.id))
        return queries_to_send
  
    def on_response_received(self, node_, rtt, nodes):
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except(IndexError):
            return # Got a response from myself. Just ignore it.
        m_bucket = sbucket.main
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            logger.debug('node in main')
            # node in routing table: update
            self._update_rnode_on_response_received(rnode, rtt)
            if self._maintenance_mode == NORMAL_MODE:
                m_bucket.last_changed_ts = time.time()
            if node_ in self._pinged_q_rnodes:
                logger.debug('remove from questionable')
                rnode.questionable = False
                # This node is questionable. This response proves that it is
                # alive. Remove it from the questionable dict.
                del self._pinged_q_rnodes[node_]
            return

        # The node is not in main
        if m_bucket.there_is_room():
            logger.debug('node not in main, there is room')
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.num_rnodes += 1
            self._update_rnode_on_response_received(rnode, rtt)
            if self._maintenance_mode == NORMAL_MODE:
                m_bucket.last_changed_ts = time.time()
            return
        # The main bucket is full

        # if there is a bad node inside the bucket,
        # replace it with the sending node_
        logger.debug('node not in main, no room')
        bad_rnode = self._pop_bad_rnode(m_bucket)
        if bad_rnode:
            logger.debug('there is a bad rnode')
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            # No need to update table
            self.table.num_rnodes += 0
            self._update_rnode_on_response_received(rnode, rtt)
            if self._maintenance_mode == NORMAL_MODE:
                m_bucket.last_changed_ts = time.time()
            return

        # There are no bad nodes. Ping questionable nodes (if any)
        logger.debug('no bad nodes, ping questionable nodes')
        q_rnodes = self._get_questionable_rnodes(m_bucket)
        queries_to_send = []
        for q_rnode in q_rnodes:
            # (0 timeouts so far, candidate node)
            c_rnode = node_.get_rnode(log_distance)
            self._update_rnode_on_response_received(c_rnode, rtt)
            self._pinged_q_rnodes[q_rnode] = [0, c_rnode]
            queries_to_send.append(message.OutgoingPingQuery(node_,
                                                             self.my_node.id))
        return queries_to_send
 
    def _pop_bad_rnode(self, mbucket):
        for rnode in mbucket.rnodes:
            if rnode.timeouts_in_a_row() >= 2:
                mbucket.remove(rnode)
                return rnode

    def _get_questionable_rnodes(self, m_bucket):
        q_rnodes = []
        for rnode in m_bucket.rnodes:
            inactivity_time = time.time() - rnode.last_seen
            if (inactivity_time > REFRESH_PERIOD
                or rnode.num_responses == 0):
                is_questionable = getattr(rnode, 'questionable', False)
                if not is_questionable:
                    rnode.questionable = True
                    q_rnodes.append(rnode)
        return q_rnodes
        
    def on_error_received(self, node_addr):
        pass
    
    def on_timeout(self, node_):
        if not node_.id:
            return [] # This is a bootstrap node (just addr, no id)
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return [] # Got a timeout from myself, WTF? Just ignore.
        m_bucket = sbucket.main
        rnode = m_bucket.get_rnode(node_)

        if not rnode:
            # This node is not in the table. Nothing to do here
            return []

        # The node is in the table. Update it
        self._update_rnode_on_timeout(rnode)
        t_strikes, c_rnode = self._pinged_q_rnodes.get(node_, (None, None))
        if t_strikes is None:
            # The node is not being checked by a "questinable ping".
            return []
        elif t_strikes == 0:
            # This is the first timeout
            self._pinged_q_rnodes[node_] = (1, c_rnode)
            # Let's give it another chance
            return [message.OutgoingPingQuery(node_, self.my_node.id)]
        elif t_strikes == 1:
            # Second timeout. You're a bad node, replace if possible
            # check if the candidate node is in the routing table
            log_distance = self.my_node.log_distance(c_rnode)
            m_bucket = self.table.get_sbucket(log_distance).main
            c_rnode_in_table = m_bucket.get_rnode(c_rnode)
            if c_rnode_in_table:
                print 'questionable node replaced'
                # replace
                m_bucket.remove(rnode)
                m_bucket.add(c_rnode)
                self.table.num_rnodes += 0
        return []
        
    def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
        if not num_nodes:
            num_nodes = NODES_PER_BUCKET[log_distance]
        return self.table.get_closest_rnodes(log_distance, num_nodes,
                                             exclude_myself)

    def get_main_rnodes(self):
        return self.table.get_main_rnodes()

    def print_stats(self):
        self.table.print_stats()

    def _update_rnode_on_query_received(self, rnode):
        """Register a query from node.

        You should call this method when receiving a query from this node.

        """
        current_time = time.time()
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout += 1
        rnode.num_queries += 1
        rnode.add_event(current_time, node.QUERY)
        rnode.last_seen = current_time

    def _update_rnode_on_response_received(self, rnode, rtt):
        """Register a reply from rnode.

        You should call this method when receiving a response from this rnode.

        """
        rnode.real_rtt = rtt
        current_time = time.time()
        #rnode._reset_refresh_task()
        if rnode.in_quarantine:
            rnode.in_quarantine = \
                rnode.last_action_ts < current_time - QUARANTINE_PERIOD
                
        rnode.last_action_ts = current_time
        rnode.num_responses += 1
        rnode.add_event(time.time(), node.RESPONSE)
        rnode.last_seen = current_time

    def _update_rnode_on_timeout(self, rnode):
        """Register a timeout for this rnode.

        You should call this method when getting a timeout for this node.

        """
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout = 0
        rnode.num_timeouts += 1
        rnode.add_event(time.time(), node.TIMEOUT)

    def _worst_rnode(self, rnodes):
        max_num_timeouts = -1
        worst_rnode_so_far = None
        for rnode in rnodes:
            num_timeouots = rnode.timeouts_in_a_row()
            if num_timeouots >= max_num_timeouts:
                max_num_timeouts = num_timeouots
                worst_rnode_so_far = rnode
        return worst_rnode_so_far
Beispiel #5
0
class RoutingManager(object):
    def __init__(self, my_node, bootstrap_nodes):
        self.my_node = my_node
        #Copy the bootstrap list
        self.bootstrap_nodes = iter(bootstrap_nodes)

        self.table = RoutingTable(my_node, NODES_PER_BUCKET)
        self.ping_msg = message.OutgoingPingQuery(my_node.id)
        self.find_closest_msg = message.OutgoingFindNodeQuery(
            my_node.id, my_node.id)

        # maintenance variables
        self._next_stale_maintenance_index = 0
        self._maintenance_mode = BOOTSTRAP_MODE
        self._replacement_queue = _ReplacementQueue(self.table)
        self._query_received_queue = _QueryReceivedQueue(self.table)
        self._found_nodes_queue = _FoundNodesQueue(self.table)
        self._maintenance_tasks = [
            self._ping_a_staled_rnode,
            self._ping_a_query_received_node,
            self._ping_a_found_node,
            self._ping_a_replacement_node,
        ]

    def do_maintenance(self):
        queries_to_send = []
        maintenance_lookup_target = None
        if self._maintenance_mode == BOOTSTRAP_MODE:
            try:
                node_ = self.bootstrap_nodes.next()
                queries_to_send = [self._get_maintenance_query(node_)]
            except (StopIteration):
                maintenance_lookup_target = self.my_node.id
                self._maintenance_mode = NORMAL_MODE
        elif self._maintenance_mode == NORMAL_MODE:
            for _ in range(len(self._maintenance_tasks)):
                # We try maintenance tasks till one of them actually does work
                # or we have tried them all (whatever happens first) We loop
                # in range because I'm going to modify self._maintenance_tasks
                task = self._maintenance_tasks.pop(0)
                self._maintenance_tasks.append(task)
                node_ = task()
                if node_:
                    queries_to_send = [self._get_maintenance_query(node_)]
                    # This task did do some work. We are done here!
                    break

        return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send,
                maintenance_lookup_target)

    def _ping_a_staled_rnode(self):
        # Don't have self._next_stale_maintenance_index lower than
        # lowest_bucket

        starting_index = self._next_stale_maintenance_index
        result = None
        while not result:
            # Find a non-empty bucket
            sbucket = self.table.get_sbucket(
                self._next_stale_maintenance_index)
            m_bucket = sbucket.main
            self._next_stale_maintenance_index = (
                self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
            if m_bucket:
                rnode = m_bucket.get_stalest_rnode()
                if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
                    result = rnode
            if self._next_stale_maintenance_index == starting_index:
                # No node to be pinged in the whole table.
                break
        return result

    def _ping_a_found_node(self):
        num_pings = 1
        if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP:
            # Extra ping when bootstrapping
            num_pings += 1
        for _ in range(num_pings):
            node_ = self._found_nodes_queue.pop(0)
            if node_:
                logger.debug('pinging node found: %r', node_)
                return node_
        return

    def _ping_a_query_received_node(self):
        return self._query_received_queue.pop(0)

    def _ping_a_replacement_node(self):
        return self._replacement_queue.pop(0)

    def _get_maintenance_query(self, node_):
        if not node_.id:
            # Bootstrap nodes don't have id
            return Query(self.find_closest_msg, node_)

        if random.choice((False, True)):
            # 50% chance to send find_node with my id as target
            return Query(self.find_closest_msg, node_)

        # 50% chance to send a find_node to fill up a non-full bucket
        target_log_distance = self.table.find_next_bucket_with_room_index(
            node_=node_)
        if target_log_distance:
            target = self.my_node.id.generate_close_id(target_log_distance)
            return Query(
                message.OutgoingFindNodeQuery(self.my_node.id, target), node_)
        else:
            # Every bucket is full. We send a ping instead.
            return Query(self.ping_msg, node_)

    def on_query_received(self, node_):
        '''
        Return None when nothing to do
        Return a list of queries when queries need to be sent (the queries
        will be sent out by the caller)
        '''
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return  # Got a query from myself. Just ignore it.

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: inform rnode
            self._update_rnode_on_query_received(rnode)
            return

        # node is not in the routing table
        if m_bucket.there_is_room():
            # There is room in the bucket: queue it
            self._query_received_queue.add(node_, log_distance)
            return
        # No room in the main routing table
        # Add to replacement table (if the bucket is not full)
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode \
                and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_query_received(rnode)
        return

    def on_response_received(self, node_, rtt, nodes):
        if nodes:
            logger.debug('nodes found: %r', nodes)
        self._found_nodes_queue.add(nodes)

        logger.debug('on response received %f', rtt)
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return  # Got a response from myself. Just ignore it.
        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: update
            self._update_rnode_on_response_received(rnode, rtt)
            return
        # The node is not in main
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            # node in replacement table
            # let's see whether there is room in the main
            self._update_rnode_on_response_received(rnode, rtt)
            #TODO: leave this for the maintenance task
            if m_bucket.there_is_room():
                m_bucket.add(rnode)
                self.table.update_lowest_index(log_distance)
                self.table.num_rnodes += 1
                self._update_rnode_on_response_received(rnode, rtt)
                r_bucket.remove(rnode)
            return
        # The node is nowhere
        # Add to main table (if the bucket is not full)
        #TODO: check whether in replacement_mode
        if m_bucket.there_is_room():
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes += 1
            self._update_rnode_on_response_received(rnode, rtt)
            return
        # The main bucket is full

        # Get the worst node in replacement bucket and see whether
        # it's bad enough to be replaced by node_
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode \
                and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            # This node is better candidate than worst_rnode
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_response_received(rnode, rtt)
        return

    def on_error_received(self, node_):
        pass

    def on_timeout(self, node_):
        if not node_.id:
            return  # This is a bootstrap node (just addr, no id)
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return  # Got a timeout from myself, WTF? Just ignore.
        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: kick it out
            self._update_rnode_on_timeout(rnode)
            m_bucket.remove(rnode)
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes -= 1

            for r_rnode in r_bucket.sorted_by_rtt():
                self._replacement_queue.add(r_rnode)
            if r_bucket.there_is_room():
                r_bucket.add(rnode)
            else:
                worst_rnode = self._worst_rnode(r_bucket.rnodes)
                if worst_rnode:
                    # Replace worst node in replacement table
                    r_bucket.remove(worst_rnode)
                    r_bucket.add(rnode)
        # Node is not in main table
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            # Node in replacement table: just update rnode
            self._update_rnode_on_timeout(rnode)

    def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
        if not num_nodes:
            num_nodes = NODES_PER_BUCKET[log_distance]
        return self.table.get_closest_rnodes(log_distance, num_nodes,
                                             exclude_myself)

    def get_main_rnodes(self):
        return self.table.get_main_rnodes()

    def print_stats(self):
        self.table.print_stats()

    def _update_rnode_on_query_received(self, rnode):
        """Register a query from node.

        You should call this method when receiving a query from this node.

        """
        current_time = time.time()
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout += 1
        rnode.num_queries += 1
        rnode.add_event(current_time, node.QUERY)
        rnode.last_seen = current_time

    def _update_rnode_on_response_received(self, rnode, rtt):
        """Register a reply from rnode.

        You should call this method when receiving a response from this rnode.

        """
        rnode.rtt = rtt
        current_time = time.time()
        #rnode._reset_refresh_task()
        if rnode.in_quarantine:
            rnode.in_quarantine = \
                rnode.last_action_ts < current_time - QUARANTINE_PERIOD

        rnode.last_action_ts = current_time
        rnode.num_responses += 1
        rnode.add_event(time.time(), node.RESPONSE)
        rnode.last_seen = current_time

    def _update_rnode_on_timeout(self, rnode):
        """Register a timeout for this rnode.

        You should call this method when getting a timeout for this node.

        """
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout = 0
        rnode.num_timeouts += 1
        rnode.add_event(time.time(), node.TIMEOUT)

    def _worst_rnode(self, rnodes):
        max_num_timeouts = -1
        worst_rnode_so_far = None
        for rnode in rnodes:
            num_timeouots = rnode.timeouts_in_a_row()
            if num_timeouots >= max_num_timeouts:
                max_num_timeouts = num_timeouots
                worst_rnode_so_far = rnode
        return worst_rnode_so_far
Beispiel #6
0
class RoutingManager(object):
    def __init__(self, my_node, bootstrap_nodes, msg_f):
        self.my_node = my_node
        self.bootstrapper = bootstrap.OverlayBootstrapper(
            my_node.id, bootstrap_nodes, msg_f)
        self.msg_f = msg_f
        self.table = RoutingTable(my_node, NODES_PER_BUCKET)
        # maintenance variables
        self._next_stale_maintenance_index = 0
        self._maintenance_mode = BOOTSTRAP_MODE
        self._replacement_queue = _ReplacementQueue(self.table)
        self._query_received_queue = _QueryReceivedQueue(self.table)
        self._found_nodes_queue = _FoundNodesQueue(self.table)
        self._maintenance_tasks = [
            self._ping_a_staled_rnode,
            self._ping_a_query_received_node,
            self._ping_a_found_node,
            self._ping_a_replacement_node,
        ]
        self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
        self._num_timeouts_in_a_row = 0

    def _get_maintenance_lookup(self, lookup_target=None, nodes=[]):
        if not lookup_target:
            lookup_target = identifier.RandomId()
        if not nodes:
            log_distance = lookup_target.distance(self.my_node.id).log
            nodes = self.get_closest_rnodes(log_distance, 0, True)
        return lookup_target, nodes

    def do_maintenance(self):
        queries_to_send = []
        maintenance_lookup = None
        maintenance_delay = 0
        if self._maintenance_mode == BOOTSTRAP_MODE:
            (queries_to_send, maintenance_lookup,
             bootstrap_delay) = self.bootstrapper.do_bootstrap(
                 self.table.num_rnodes)
            if bootstrap_delay:
                maintenance_delay = bootstrap_delay
            else:
                self._maintenance_mode = FILL_BUCKETS
                self.bootstrapper.bootstrap_done()
        elif self._maintenance_mode == FILL_BUCKETS:
            if self._num_pending_filling_lookups:
                self._num_pending_filling_lookups -= 1
                maintenance_lookup = self._get_maintenance_lookup()
            else:
                self._maintenance_mode = NORMAL_MODE
        elif self._maintenance_mode == NORMAL_MODE:
            for _ in range(len(self._maintenance_tasks)):
                # We try maintenance tasks till one of them actually does work
                # or we have tried them all (whatever happens first) We loop
                # in range because I'm going to modify self._maintenance_tasks
                task = self._maintenance_tasks.pop(0)
                self._maintenance_tasks.append(task)
                node_ = task()
                if node_:
                    queries_to_send.append(self._get_maintenance_query(node_))
                    # This task did do some work. We are done here!
                    break
        if self.table.num_rnodes < MIN_RNODES:
            # Ping more found nodes when routing table has few nodes
            node_ = self._ping_a_found_node()
            if node_:
                queries_to_send.append(
                    self._get_maintenance_query(node_, do_fill_up=True))
        if not maintenance_delay:
            maintenance_delay = _MAINTENANCE_DELAY[self._maintenance_mode]
        return (maintenance_delay, queries_to_send, maintenance_lookup)

    def _ping_a_staled_rnode(self):
        starting_index = self._next_stale_maintenance_index
        result = None
        while not result:
            # Find a non-empty bucket
            sbucket = self.table.get_sbucket(
                self._next_stale_maintenance_index)
            m_bucket = sbucket.main
            self._next_stale_maintenance_index = (
                self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
            if m_bucket:
                rnode = m_bucket.get_stalest_rnode()
                if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
                    result = rnode
            if self._next_stale_maintenance_index == starting_index:
                # No node to be pinged in the whole table.
                break
        return result

    def _ping_a_found_node(self):
        node_ = self._found_nodes_queue.pop(0)
        if node_:
            logger.debug('pinging node found: %r', node_)
        return node_

    def _ping_a_query_received_node(self):
        return self._query_received_queue.pop(0)

    def _ping_a_replacement_node(self):
        return self._replacement_queue.pop(0)

    def _get_maintenance_query(self, node_, do_fill_up=False):
        '''
        if not node_.id:
            # Bootstrap nodes don't have id
            return message.OutgoingFindNodeQuery(node_,
                                                 self.my_node.id,
                                                 self.my_node.id, None)
        '''
        if do_fill_up or random.choice((False, True)):

            # 50% chance to send a find_node to fill up a non-full bucket
            target_log_distance = self.table.find_next_bucket_with_room_index(
                node_=node_)
            if target_log_distance:
                target = self.my_node.id.generate_close_id(target_log_distance)
                msg = self.msg_f.outgoing_find_node_query(node_, target, None)
            else:
                # Every bucket is full. We send a ping instead.
                msg = self.msg_f.outgoing_ping_query(node_)
        else:
            # 50% chance to send find_node with my id as target
            msg = self.msg_f.outgoing_find_node_query(node_, self.my_node.id,
                                                      None)
        return msg

    def on_query_received(self, node_):
        '''
        Return None when nothing to do
        Return a list of queries when queries need to be sent (the queries
        will be sent out by the caller)
        '''
        self._num_timeouts_in_a_row = 0
        if self.bootstrapper.is_bootstrap_node(node_):
            return

        log_distance = self.my_node.distance(node_).log
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return  # Got a query from myself. Just ignore it.

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        if node_.ip in m_bucket.ips_in_table:
            rnode = m_bucket.get_rnode(node_)
            if rnode:
                # node in routing table: update rnode
                self._update_rnode_on_query_received(rnode)
            # This IP is in the table. Stop here to avoid multiple entries
            # with the same IP
            return

        # Now, consider adding this node to the routing table
        if m_bucket.there_is_room():
            # There is room in the bucket: queue it
            self._query_received_queue.add(node_, log_distance)
            return
        # No room in the main routing table
        # Add to replacement table (if the bucket is not full)
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode \
                and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_query_received(rnode)
        return

    def on_response_received(self, node_, rtt, nodes):
        self._num_timeouts_in_a_row = 0
        if self.bootstrapper.is_bootstrap_node(node_):
            return

        if nodes:
            logger.debug('nodes found: %r', nodes)
        self._found_nodes_queue.add(nodes)

        logger.debug('on response received %f', rtt)
        log_distance = self.my_node.distance(node_).log
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return  # Got a response from myself. Just ignore it.
        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if node_.ip in m_bucket.ips_in_table:
            rnode = m_bucket.get_rnode(node_)
            if rnode:
                # node in routing table: update rnode
                self._update_rnode_on_response_received(rnode, rtt)
            # This IP is in the table. Stop here to avoid multiple entries
            # with the same IP
            return

        # Now, consider adding this node to the routing table
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            # node in replacement table
            # let's see whether there is room in the main
            self._update_rnode_on_response_received(rnode, rtt)
            # TODO: leave this for the maintenance task
            if m_bucket.there_is_room():
                m_bucket.add(rnode)
                self.table.num_rnodes += 1
                self._update_rnode_on_response_received(rnode, rtt)
                r_bucket.remove(rnode)
            return
        # The node is nowhere
        # Add to main table (if the bucket is not full)
        # TODO: check whether in replacement_mode
        if m_bucket.there_is_room():
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.num_rnodes += 1
            self._update_rnode_on_response_received(rnode, rtt)
            return
        # The main bucket is full
        # Let's see whether this node's latency is good
        current_time = time.time()
        rnode_to_be_replaced = None
        m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True)
        for rnode in m_bucket.rnodes:
            rnode_age = current_time - rnode.bucket_insertion_ts
            if rtt < rnode.rtt * (1 - (rnode_age / 7200)):
                # A rnode can only be replaced when the candidate node's RTT
                # is shorter by a factor. Over time, this factor
                # decreases. For instance, when rnode has been in the bucket
                # for 30 mins (1800 secs), a candidate's RTT must be at most
                # 25% of the rnode's RTT (ie. two times faster). After two
                # hours, a rnode cannot be replaced by this method.
                #                print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % (
                #                rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts)
                rnode_to_be_replaced = rnode
                break
        if rnode_to_be_replaced:
            m_bucket.remove(rnode_to_be_replaced)
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            # No need to update table
            self.table.num_rnodes += 0
            self._update_rnode_on_response_received(rnode, rtt)
            return

        # Get the worst node in replacement bucket and see whether
        # it's bad enough to be replaced by node_
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode \
                and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            # This node is better candidate than worst_rnode
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_response_received(rnode, rtt)
        return

    def on_error_received(self, node_addr):
        # if self.bootstrapper.is_bootstrap_node(node_):
        #     return
        return

    def on_timeout(self, node_):
        self._num_timeouts_in_a_row += 1
        if self._num_timeouts_in_a_row > MAX_TIMEOUTS_IN_A_ROW:
            # stop, do not expell nodes from routing table
            return []
        if self.bootstrapper.is_bootstrap_node(node_):
            return []

        log_distance = self.my_node.distance(node_).log
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return []  # Got a timeout from myself, WTF? Just ignore.
        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: kick it out
            self._update_rnode_on_timeout(rnode)
            m_bucket.remove(rnode)
            self.table.num_rnodes -= 1

            for r_rnode in r_bucket.sorted_by_rtt():
                self._replacement_queue.add(r_rnode)
            if r_bucket.there_is_room():
                r_bucket.add(rnode)
            else:
                worst_rnode = self._worst_rnode(r_bucket.rnodes)
                if worst_rnode:
                    # Replace worst node in replacement table
                    r_bucket.remove(worst_rnode)
                    r_bucket.add(rnode)
        # Node is not in main table
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            # Node in replacement table: just update rnode
            self._update_rnode_on_timeout(rnode)
        return []

    def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
        if not num_nodes:
            num_nodes = NODES_PER_BUCKET[log_distance]
        return self.table.get_closest_rnodes(log_distance, num_nodes,
                                             exclude_myself)

    def get_main_rnodes(self):
        return self.table.get_main_rnodes()

    def print_stats(self):
        self.table.print_stats()

    def _update_rnode_on_query_received(self, rnode):
        """Register a query from node.

        You should call this method when receiving a query from this node.

        """
        current_time = time.time()
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout += 1
        rnode.num_queries += 1
        rnode.add_event(current_time, node.QUERY)
        rnode.last_seen = current_time

    def _update_rnode_on_response_received(self, rnode, rtt):
        """Register a reply from rnode.

        You should call this method when receiving a response from this rnode.

        """
        rnode.rtt = rtt
        current_time = time.time()
        # rnode._reset_refresh_task()
        if rnode.in_quarantine:
            rnode.in_quarantine = \
                rnode.last_action_ts < current_time - QUARANTINE_PERIOD

        rnode.last_action_ts = current_time
        rnode.num_responses += 1
        rnode.add_event(time.time(), node.RESPONSE)
        rnode.last_seen = current_time

    def _update_rnode_on_timeout(self, rnode):
        """Register a timeout for this rnode.

        You should call this method when getting a timeout for this node.

        """
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout = 0
        rnode.num_timeouts += 1
        rnode.add_event(time.time(), node.TIMEOUT)

    def _worst_rnode(self, rnodes):
        max_num_timeouts = -1
        worst_rnode_so_far = None
        for rnode in rnodes:
            num_timeouots = rnode.timeouts_in_a_row()
            if num_timeouots >= max_num_timeouts:
                max_num_timeouts = num_timeouots
                worst_rnode_so_far = rnode
        return worst_rnode_so_far
class RoutingManager(object):
    
    def __init__(self, my_node, bootstrap_nodes, msg_f):
        self.my_node = my_node
        self.bootstrapper = bootstrap.OverlayBootstrapper(my_node.id,
                                                          bootstrap_nodes, msg_f)
        self.msg_f = msg_f
        self.table = RoutingTable(my_node, NODES_PER_BUCKET)
        # maintenance variables
        self._next_stale_maintenance_index = 0
        self._maintenance_mode = BOOTSTRAP_MODE
        self._replacement_queue = _ReplacementQueue(self.table)
        self._query_received_queue = _QueryReceivedQueue(self.table)
        self._found_nodes_queue = _FoundNodesQueue(self.table)
        self._maintenance_tasks = [self._ping_a_staled_rnode,
                                   self._ping_a_query_received_node,
                                   self._ping_a_found_node,
                                   self._ping_a_replacement_node,
                                   ]
        self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS

    def _get_maintenance_lookup(self, lookup_target=None, nodes=[]):
        if not lookup_target:
            lookup_target = identifier.RandomId()
        if not nodes:
            log_distance = lookup_target.log_distance(self.my_node.id)
            nodes = self.get_closest_rnodes(log_distance, 0, True)
        return lookup_target, nodes
        
                
    def do_maintenance(self):
        queries_to_send = []
        maintenance_lookup = None
        maintenance_delay = 0
        if self._maintenance_mode == BOOTSTRAP_MODE: 
                (queries_to_send,
                 maintenance_lookup,
                 bootstrap_delay) = self.bootstrapper.do_bootstrap(
                    self.table.num_rnodes)
                if bootstrap_delay:
                    maintenance_delay = bootstrap_delay
                else:
                    self._maintenance_mode = FILL_BUCKETS
        elif self._maintenance_mode == FILL_BUCKETS:
            if self._num_pending_filling_lookups:
                self._num_pending_filling_lookups -= 1
                maintenance_lookup = self._get_maintenance_lookup()
            else:
                self._maintenance_mode = NORMAL_MODE
        elif self._maintenance_mode == NORMAL_MODE:
            for _ in range(len(self._maintenance_tasks)):
                # We try maintenance tasks till one of them actually does work
                # or we have tried them all (whatever happens first) We loop
                # in range because I'm going to modify self._maintenance_tasks
                task = self._maintenance_tasks.pop(0)
                self._maintenance_tasks.append(task)
                node_ = task()
                if node_:
                    queries_to_send.append(self._get_maintenance_query(node_))
                    # This task did do some work. We are done here!
                    break
        if self.table.num_rnodes < MIN_RNODES:
            # Ping more found nodes when routing table has few nodes
            node_ = self._ping_a_found_node()
            if node_:
                queries_to_send.append(self._get_maintenance_query(
                        node_, do_fill_up=True))
        if not maintenance_delay:
            maintenance_delay = _MAINTENANCE_DELAY[self._maintenance_mode]
        return (maintenance_delay, queries_to_send, maintenance_lookup)

    def _ping_a_staled_rnode(self):
        starting_index = self._next_stale_maintenance_index
        result = None
        while not result:
            # Find a non-empty bucket
            sbucket = self.table.get_sbucket(
                self._next_stale_maintenance_index)
            m_bucket = sbucket.main
            self._next_stale_maintenance_index = (
                self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
            if m_bucket:
                rnode = m_bucket.get_stalest_rnode()
                if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
                    result = rnode
            if self._next_stale_maintenance_index == starting_index:
                # No node to be pinged in the whole table.
                break
        return result

    def _ping_a_found_node(self):
        node_ = self._found_nodes_queue.pop(0)
        if node_:
            logger.debug('pinging node found: %r', node_)
        return node_
        
    def _ping_a_query_received_node(self):
        return self._query_received_queue.pop(0)

    def _ping_a_replacement_node(self):
        return self._replacement_queue.pop(0)
                                  
    def _get_maintenance_query(self, node_, do_fill_up=False):
        '''
        if not node_.id: 
            # Bootstrap nodes don't have id
            return message.OutgoingFindNodeQuery(node_,
                                                 self.my_node.id,
                                                 self.my_node.id, None)
        '''
        if do_fill_up or random.choice((False, True)):

            # 50% chance to send a find_node to fill up a non-full bucket
            target_log_distance = self.table.find_next_bucket_with_room_index(
                node_=node_)
            if target_log_distance:
                target = self.my_node.id.generate_close_id(target_log_distance)
                msg = self.msg_f.outgoing_find_node_query(node_,
                                                          target, None)
            else:
                # Every bucket is full. We send a ping instead.
                msg = self.msg_f.outgoing_ping_query(node_)
        else:
            # 50% chance to send find_node with my id as target
            msg = self.msg_f.outgoing_find_node_query(node_,
                                                      self.my_node.id, None)
        return msg
        
    def on_query_received(self, node_):
        '''
        Return None when nothing to do
        Return a list of queries when queries need to be sent (the queries
        will be sent out by the caller)
        '''
        if self.bootstrapper.is_bootstrap_node(node_):
            return
        
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except(IndexError):
            return # Got a query from myself. Just ignore it.

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: inform rnode
            self._update_rnode_on_query_received(rnode)
            return
        
        # node is not in the routing table
        if m_bucket.there_is_room():
            # There is room in the bucket: queue it
            self._query_received_queue.add(node_, log_distance)
            return
        # No room in the main routing table
        # Add to replacement table (if the bucket is not full)
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode \
                and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_query_received(rnode)
        return
            
    def on_response_received(self, node_, rtt, nodes):
        if self.bootstrapper.is_bootstrap_node(node_):
            return

        if nodes:
            logger.debug('nodes found: %r', nodes)
        self._found_nodes_queue.add(nodes)

        logger.debug('on response received %f', rtt)
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except(IndexError):
            return # Got a response from myself. Just ignore it.
        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: update
            self._update_rnode_on_response_received(rnode, rtt)
            return
        # The node is not in main
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            # node in replacement table
            # let's see whether there is room in the main
            self._update_rnode_on_response_received(rnode, rtt)
            #TODO: leave this for the maintenance task
            if m_bucket.there_is_room():
                m_bucket.add(rnode)
                self.table.num_rnodes += 1
                self._update_rnode_on_response_received(rnode, rtt)
                r_bucket.remove(rnode)
            return
        # The node is nowhere
        # Add to main table (if the bucket is not full)
        #TODO: check whether in replacement_mode
        if m_bucket.there_is_room():
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.num_rnodes += 1
            self._update_rnode_on_response_received(rnode, rtt)
            return
        # The main bucket is full
        # Let's see whether this node's latency is good
        current_time = time.time()
        rnode_to_be_replaced = None
        m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True)
        for rnode in m_bucket.rnodes:
            rnode_age = current_time - rnode.bucket_insertion_ts
            if rtt < rnode.rtt * (1 - (rnode_age / 7200)):
                # A rnode can only be replaced when the candidate node's RTT
                # is shorter by a factor. Over time, this factor
                # decreases. For instance, when rnode has been in the bucket
                # for 30 mins (1800 secs), a candidate's RTT must be at most
                # 25% of the rnode's RTT (ie. two times faster). After two
                # hours, a rnode cannot be replaced by this method.
#                print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % (
#                rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts)
                rnode_to_be_replaced = rnode
                break
        if rnode_to_be_replaced:
            m_bucket.remove(rnode_to_be_replaced)
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            # No need to update table
            self.table.num_rnodes += 0
            self._update_rnode_on_response_received(rnode, rtt)
            return
            
        # Get the worst node in replacement bucket and see whether
        # it's bad enough to be replaced by node_
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode \
                and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            # This node is better candidate than worst_rnode
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_response_received(rnode, rtt)
        return
        
    def on_error_received(self, node_addr):
        if self.bootstrapper.is_bootstrap_node(node_):
            return

        return
    
    def on_timeout(self, node_):
        if self.bootstrapper.is_bootstrap_node(node_):
            return

        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except (IndexError):
            return [] # Got a timeout from myself, WTF? Just ignore.
        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            # node in routing table: kick it out
            self._update_rnode_on_timeout(rnode)
            m_bucket.remove(rnode)
            self.table.num_rnodes -= 1

            for r_rnode in r_bucket.sorted_by_rtt():
                self._replacement_queue.add(r_rnode)
            if r_bucket.there_is_room():
                r_bucket.add(rnode)
            else:
                worst_rnode = self._worst_rnode(r_bucket.rnodes)
                if worst_rnode:
                    # Replace worst node in replacement table
                    r_bucket.remove(worst_rnode)
                    r_bucket.add(rnode)
        # Node is not in main table
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            # Node in replacement table: just update rnode
            self._update_rnode_on_timeout(rnode)
        return []
            
    def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
        if not num_nodes:
            num_nodes = NODES_PER_BUCKET[log_distance]
        return self.table.get_closest_rnodes(log_distance, num_nodes,
                                             exclude_myself)

    def get_main_rnodes(self):
        return self.table.get_main_rnodes()

    def print_stats(self):
        self.table.print_stats()

    def _update_rnode_on_query_received(self, rnode):
        """Register a query from node.

        You should call this method when receiving a query from this node.

        """
        current_time = time.time()
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout += 1
        rnode.num_queries += 1
        rnode.add_event(current_time, node.QUERY)
        rnode.last_seen = current_time

    def _update_rnode_on_response_received(self, rnode, rtt):
        """Register a reply from rnode.

        You should call this method when receiving a response from this rnode.

        """
        rnode.rtt = rtt
        current_time = time.time()
        #rnode._reset_refresh_task()
        if rnode.in_quarantine:
            rnode.in_quarantine = \
                rnode.last_action_ts < current_time - QUARANTINE_PERIOD
                
        rnode.last_action_ts = current_time
        rnode.num_responses += 1
        rnode.add_event(time.time(), node.RESPONSE)
        rnode.last_seen = current_time

    def _update_rnode_on_timeout(self, rnode):
        """Register a timeout for this rnode.

        You should call this method when getting a timeout for this node.

        """
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout = 0
        rnode.num_timeouts += 1
        rnode.add_event(time.time(), node.TIMEOUT)

    def _worst_rnode(self, rnodes):
        max_num_timeouts = -1
        worst_rnode_so_far = None
        for rnode in rnodes:
            num_timeouots = rnode.timeouts_in_a_row()
            if num_timeouots >= max_num_timeouts:
                max_num_timeouts = num_timeouots
                worst_rnode_so_far = rnode
        return worst_rnode_so_far
Beispiel #8
0
class RoutingManager(object):
    def __init__(self, my_node, bootstrap_nodes):
        self.my_node = my_node
        self.bootstrap_nodes = iter(bootstrap_nodes)
        self.table = RoutingTable(my_node, NODES_PER_BUCKET)
        self.ping_msg = message.OutgoingPingQuery(my_node.id)
        self.find_closest_msg = message.OutgoingFindNodeQuery(
            my_node.id, my_node.id)
        self._next_stale_maintenance_index = 0
        self._maintenance_mode = BOOTSTRAP_MODE
        self._replacement_queue = _ReplacementQueue(self.table)
        self._query_received_queue = _QueryReceivedQueue(self.table)
        self._found_nodes_queue = _FoundNodesQueue(self.table)
        self._maintenance_tasks = [
            self._ping_a_staled_rnode, self._ping_a_query_received_node,
            self._ping_a_found_node, self._ping_a_replacement_node
        ]

    def do_maintenance(self):
        queries_to_send = []
        maintenance_lookup_target = None
        if self._maintenance_mode == BOOTSTRAP_MODE:
            try:
                node_ = self.bootstrap_nodes.next()
                queries_to_send = [self._get_maintenance_query(node_)]
            except StopIteration:
                maintenance_lookup_target = self.my_node.id
                self._maintenance_mode = NORMAL_MODE

        elif self._maintenance_mode == NORMAL_MODE:
            for _ in range(len(self._maintenance_tasks)):
                task = self._maintenance_tasks.pop(0)
                self._maintenance_tasks.append(task)
                node_ = task()
                if node_:
                    queries_to_send = [self._get_maintenance_query(node_)]
                    break

        return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send,
                maintenance_lookup_target)

    def _ping_a_staled_rnode(self):
        starting_index = self._next_stale_maintenance_index
        result = None
        while not result:
            sbucket = self.table.get_sbucket(
                self._next_stale_maintenance_index)
            m_bucket = sbucket.main
            self._next_stale_maintenance_index = (
                self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
            if m_bucket:
                rnode = m_bucket.get_stalest_rnode()
                if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
                    result = rnode
            if self._next_stale_maintenance_index == starting_index:
                break

        return result

    def _ping_a_found_node(self):
        num_pings = 1
        if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP:
            num_pings += 1
        for _ in range(num_pings):
            node_ = self._found_nodes_queue.pop(0)
            if node_:
                logger.debug('pinging node found: %r', node_)
                return node_

    def _ping_a_query_received_node(self):
        return self._query_received_queue.pop(0)

    def _ping_a_replacement_node(self):
        return self._replacement_queue.pop(0)

    def _get_maintenance_query(self, node_):
        if not node_.id:
            return Query(self.find_closest_msg, node_)
        elif random.choice((False, True)):
            return Query(self.find_closest_msg, node_)
        target_log_distance = self.table.find_next_bucket_with_room_index(
            node_=node_)
        if target_log_distance:
            target = self.my_node.id.generate_close_id(target_log_distance)
            return Query(
                message.OutgoingFindNodeQuery(self.my_node.id, target), node_)
        else:
            return Query(self.ping_msg, node_)

    def on_query_received(self, node_):
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except IndexError:
            return

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_query_received(rnode)
            return
        if m_bucket.there_is_room():
            self._query_received_queue.add(node_, log_distance)
            return
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_query_received(rnode)

    def on_response_received(self, node_, rtt, nodes):
        if nodes:
            logger.debug('nodes found: %r', nodes)
        self._found_nodes_queue.add(nodes)
        logger.debug('on response received %f', rtt)
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except IndexError:
            return

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_response_received(rnode, rtt)
            return
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_response_received(rnode, rtt)
            if m_bucket.there_is_room():
                m_bucket.add(rnode)
                self.table.update_lowest_index(log_distance)
                self.table.num_rnodes += 1
                self._update_rnode_on_response_received(rnode, rtt)
                r_bucket.remove(rnode)
            return
        if m_bucket.there_is_room():
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes += 1
            self._update_rnode_on_response_received(rnode, rtt)
            return
        current_time = time.time()
        rnode_to_be_replaced = None
        for rnode in reversed(m_bucket.rnodes):
            rnode_age = current_time - rnode.bucket_insertion_ts
            if rtt < rnode.rtt * (1 - rnode_age / 7200):
                rnode_to_be_replaced = rnode
                break

        if rnode_to_be_replaced:
            m_bucket.remove(rnode_to_be_replaced)
            rnode = node_.get_rnode(log_distance)
            m_bucket.add(rnode)
            self.table.num_rnodes += 0
            self._update_rnode_on_response_received(rnode, rtt)
            return
        worst_rnode = self._worst_rnode(r_bucket.rnodes)
        if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
            r_bucket.remove(worst_rnode)
            rnode = node_.get_rnode(log_distance)
            r_bucket.add(rnode)
            self._update_rnode_on_response_received(rnode, rtt)

    def on_error_received(self, node_addr):
        pass

    def on_timeout(self, node_):
        if not node_.id:
            return
        log_distance = self.my_node.log_distance(node_)
        try:
            sbucket = self.table.get_sbucket(log_distance)
        except IndexError:
            return

        m_bucket = sbucket.main
        r_bucket = sbucket.replacement
        rnode = m_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_timeout(rnode)
            m_bucket.remove(rnode)
            self.table.update_lowest_index(log_distance)
            self.table.num_rnodes -= 1
            for r_rnode in r_bucket.sorted_by_rtt():
                self._replacement_queue.add(r_rnode)

            if r_bucket.there_is_room():
                r_bucket.add(rnode)
            else:
                worst_rnode = self._worst_rnode(r_bucket.rnodes)
                if worst_rnode:
                    r_bucket.remove(worst_rnode)
                    r_bucket.add(rnode)
        rnode = r_bucket.get_rnode(node_)
        if rnode:
            self._update_rnode_on_timeout(rnode)

    def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
        if not num_nodes:
            num_nodes = NODES_PER_BUCKET[log_distance]
        return self.table.get_closest_rnodes(log_distance, num_nodes,
                                             exclude_myself)

    def get_main_rnodes(self):
        return self.table.get_main_rnodes()

    def print_stats(self):
        self.table.print_stats()

    def _update_rnode_on_query_received(self, rnode):
        current_time = time.time()
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout += 1
        rnode.num_queries += 1
        rnode.add_event(current_time, node.QUERY)
        rnode.last_seen = current_time

    def _update_rnode_on_response_received(self, rnode, rtt):
        rnode.rtt = rtt
        current_time = time.time()
        if rnode.in_quarantine:
            rnode.in_quarantine = rnode.last_action_ts < current_time - QUARANTINE_PERIOD
        rnode.last_action_ts = current_time
        rnode.num_responses += 1
        rnode.add_event(time.time(), node.RESPONSE)
        rnode.last_seen = current_time

    def _update_rnode_on_timeout(self, rnode):
        rnode.last_action_ts = time.time()
        rnode.msgs_since_timeout = 0
        rnode.num_timeouts += 1
        rnode.add_event(time.time(), node.TIMEOUT)

    def _worst_rnode(self, rnodes):
        max_num_timeouts = -1
        worst_rnode_so_far = None
        for rnode in rnodes:
            num_timeouots = rnode.timeouts_in_a_row()
            if num_timeouots >= max_num_timeouts:
                max_num_timeouts = num_timeouots
                worst_rnode_so_far = rnode

        return worst_rnode_so_far