def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
#Copy the bootstrap list
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._maintenance_mode = BOOTSTRAP_MODE
self._pinged_q_rnodes = {} # questionable nodes which have been
# recently pinged
self._maintenance_tasks = [
self._refresh_stale_bucket,
#self._ping_a_staled_rnode,
# self._ping_a_query_received_node,
# self._ping_a_found_node,
]
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
self.ping_msg = message.OutgoingPingQuery(my_node.id)
self.find_closest_msg = message.OutgoingFindNodeQuery(
my_node.id, my_node.id)
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [
self._ping_a_staled_rnode, self._ping_a_query_received_node,
self._ping_a_found_node, self._ping_a_replacement_node
]
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
#Copy the bootstrap list
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [
self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
def __init__(self, my_node, msg_f, bootstrapper):
self.my_node = my_node
self.msg_f = msg_f
self.bootstrapper = bootstrapper
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = FILL_BUCKETS
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [
self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
self._num_timeouts_in_a_row = 0
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
#Copy the bootstrap list
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._maintenance_mode = BOOTSTRAP_MODE
self._pinged_q_rnodes = {} # questionable nodes which have been
# recently pinged
self._maintenance_tasks = [self._refresh_stale_bucket,
#self._ping_a_staled_rnode,
# self._ping_a_query_received_node,
# self._ping_a_found_node,
]
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
self.ping_msg = message.OutgoingPingQuery(my_node.id)
self.find_closest_msg = message.OutgoingFindNodeQuery(my_node.id, my_node.id)
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node]
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
self.bootstrapper = bootstrap.OverlayBootstrapper(my_node.id,
bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
#Copy the bootstrap list
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
def __init__(self, my_node, msg_f, bootstrapper):
self.my_node = my_node
self.msg_f = msg_f
self.bootstrapper = bootstrapper
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = FILL_BUCKETS
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
self._num_timeouts_in_a_row = 0
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
self.ping_msg = message.OutgoingPingQuery(my_node.id)
self.find_closest_msg = message.OutgoingFindNodeQuery(my_node.id, my_node.id)
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node]
def do_maintenance(self):
queries_to_send = []
maintenance_lookup_target = None
if self._maintenance_mode == BOOTSTRAP_MODE:
try:
node_ = self.bootstrap_nodes.next()
queries_to_send = [self._get_maintenance_query(node_)]
except StopIteration:
maintenance_lookup_target = self.my_node.id
self._maintenance_mode = NORMAL_MODE
elif self._maintenance_mode == NORMAL_MODE:
for _ in range(len(self._maintenance_tasks)):
task = self._maintenance_tasks.pop(0)
self._maintenance_tasks.append(task)
node_ = task()
if node_:
queries_to_send = [self._get_maintenance_query(node_)]
break
return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send, maintenance_lookup_target)
def _ping_a_staled_rnode(self):
starting_index = self._next_stale_maintenance_index
result = None
while not result:
sbucket = self.table.get_sbucket(self._next_stale_maintenance_index)
m_bucket = sbucket.main
self._next_stale_maintenance_index = (self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
if m_bucket:
rnode = m_bucket.get_stalest_rnode()
if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
result = rnode
if self._next_stale_maintenance_index == starting_index:
break
return result
def _ping_a_found_node(self):
num_pings = 1
if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP:
num_pings += 1
for _ in range(num_pings):
node_ = self._found_nodes_queue.pop(0)
if node_:
logger.debug('pinging node found: %r', node_)
return node_
def _ping_a_query_received_node(self):
return self._query_received_queue.pop(0)
def _ping_a_replacement_node(self):
return self._replacement_queue.pop(0)
def _get_maintenance_query(self, node_):
if not node_.id:
return Query(self.find_closest_msg, node_)
elif random.choice((False, True)):
return Query(self.find_closest_msg, node_)
target_log_distance = self.table.find_next_bucket_with_room_index(node_=node_)
if target_log_distance:
target = self.my_node.id.generate_close_id(target_log_distance)
return Query(message.OutgoingFindNodeQuery(self.my_node.id, target), node_)
else:
return Query(self.ping_msg, node_)
def on_query_received(self, node_):
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except IndexError:
return
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_query_received(rnode)
return
if m_bucket.there_is_room():
self._query_received_queue.add(node_, log_distance)
return
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
def on_response_received(self, node_, rtt, nodes):
if nodes:
logger.debug('nodes found: %r', nodes)
self._found_nodes_queue.add(nodes)
logger.debug('on response received %f', rtt)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except IndexError:
return
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_response_received(rnode, rtt)
return
rnode = r_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_response_received(rnode, rtt)
if m_bucket.there_is_room():
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
r_bucket.remove(rnode)
return
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
return
current_time = time.time()
rnode_to_be_replaced = None
for rnode in reversed(m_bucket.rnodes):
rnode_age = current_time - rnode.bucket_insertion_ts
if rtt < rnode.rtt * (1 - rnode_age / 7200):
rnode_to_be_replaced = rnode
break
if rnode_to_be_replaced:
m_bucket.remove(rnode_to_be_replaced)
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.num_rnodes += 0
self._update_rnode_on_response_received(rnode, rtt)
return
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
def on_error_received(self, node_addr):
pass
def on_timeout(self, node_):
if not node_.id:
return
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except IndexError:
return
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_timeout(rnode)
m_bucket.remove(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes -= 1
for r_rnode in r_bucket.sorted_by_rtt():
self._replacement_queue.add(r_rnode)
if r_bucket.there_is_room():
r_bucket.add(rnode)
else:
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode:
r_bucket.remove(worst_rnode)
r_bucket.add(rnode)
rnode = r_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_timeout(rnode)
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes, exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
rnode.rtt = rtt
current_time = time.time()
if rnode.in_quarantine:
rnode.in_quarantine = rnode.last_action_ts < current_time - QUARANTINE_PERIOD
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
#Copy the bootstrap list
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
self.ping_msg = message.OutgoingPingQuery(my_node.id)
self.find_closest_msg = message.OutgoingFindNodeQuery(
my_node.id, my_node.id)
# maintenance variables
self._maintenance_mode = BOOTSTRAP_MODE
self._pinged_q_rnodes = {} # questionable nodes which have been
# recently pinged
self._maintenance_tasks = [
self._refresh_stale_bucket,
#self._ping_a_staled_rnode,
# self._ping_a_query_received_node,
# self._ping_a_found_node,
]
def do_maintenance(self):
queries_to_send = []
maintenance_lookup_target = None
if self._maintenance_mode == BOOTSTRAP_MODE:
try:
node_ = self.bootstrap_nodes.next()
queries_to_send = [self._get_maintenance_query(node_)]
except (StopIteration):
maintenance_lookup_target = self.my_node.id
self._maintenance_mode = FIND_NODES_MODE
return (10, [], maintenance_lookup_target)
else:
maintenance_lookup_target = self._refresh_stale_bucket()
return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send,
maintenance_lookup_target)
def _refresh_stale_bucket(self):
maintenance_lookup_target = None
current_time = time.time()
for i in xrange(self.table.lowest_index, NUM_BUCKETS):
sbucket = self.table.get_sbucket(i)
m_bucket = sbucket.main
if not m_bucket:
continue
inactivity_time = current_time - m_bucket.last_changed_ts
if inactivity_time > REFRESH_PERIOD:
# print time.time(), '>>>>>>>>>>>>>>> refreshing bucket %d after %f secs' % (
# i, inactivity_time)
maintenance_lookup_target = self.my_node.id.generate_close_id(
i)
m_bucket.last_changed_ts = current_time
return maintenance_lookup_target
self._maintenance_mode = NORMAL_MODE
return None
def _get_maintenance_query(self, node_):
return Query(self.ping_msg, node_)
def on_query_received(self, node_):
'''
Return None when nothing to do
Return a list of queries when queries need to be sent (the queries
will be sent out by the caller)
'''
if self._maintenance_mode != NORMAL_MODE:
return
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a query from myself. Just ignore it.
m_bucket = sbucket.main
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: inform rnode
self._update_rnode_on_query_received(rnode)
return
# node is not in the routing table
if m_bucket.there_is_room():
# There is room in the bucket. Just add the new node.
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_query_received(rnode)
return
# No room in the main routing table
# Check whether there is a bad node to be replaced.
bad_rnode = self._pop_bad_rnode(m_bucket)
if bad_rnode:
# We have a bad node in the bucket. Replace it with the new node.
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 0
return
# No bad nodes. Check for questionable nodes
q_rnodes = self._get_questionable_rnodes(m_bucket)
queries_to_send = []
# if q_rnodes:
# print time.time(), '-----pinging questionable nodes in',
# print log_distance
# print q_rnodes
for q_rnode in q_rnodes:
# Ping questinable nodes to check whether they are still alive.
# (0 timeouts so far, candidate node)
c_rnode = node_.get_rnode(log_distance)
self._update_rnode_on_query_received(c_rnode)
self._pinged_q_rnodes[q_rnode] = [0, c_rnode]
queries_to_send.append(Query(self.ping_msg, q_rnode))
return queries_to_send
def on_response_received(self, node_, rtt, nodes):
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a response from myself. Just ignore it.
m_bucket = sbucket.main
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update
self._update_rnode_on_response_received(rnode, rtt)
if self._maintenance_mode == NORMAL_MODE:
m_bucket.last_changed_ts = time.time()
if node_ in self._pinged_q_rnodes:
# This node is questionable. This response proves that it is
# alive. Remove it from the questionable dict.
del self._pinged_q_rnodes[node_]
return
# The node is not in main
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
if self._maintenance_mode == NORMAL_MODE:
m_bucket.last_changed_ts = time.time()
return
# The main bucket is full
# if there is a bad node inside the bucket,
# replace it with the sending node_
bad_rnode = self._pop_bad_rnode(m_bucket)
if bad_rnode:
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
if self._maintenance_mode == NORMAL_MODE:
m_bucket.last_changed_ts = time.time()
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 0
return
# There are no bad nodes. Ping questionable nodes (if any)
q_rnodes = self._get_questionable_rnodes(m_bucket)
queries_to_send = []
for q_rnode in q_rnodes:
# (0 timeouts so far, candidate node)
c_rnode = node_.get_rnode(log_distance)
self._update_rnode_on_response_received(c_rnode, rtt)
self._pinged_q_rnodes[q_rnode] = [0, c_rnode]
queries_to_send.append(Query(self.ping_msg, q_rnode))
return queries_to_send
def _pop_bad_rnode(self, mbucket):
for rnode in mbucket.rnodes:
if rnode.timeouts_in_a_row() >= 2:
mbucket.remove(rnode)
return rnode
def _get_questionable_rnodes(self, m_bucket):
q_rnodes = []
for rnode in m_bucket.rnodes:
inactivity_time = time.time() - rnode.last_seen
if inactivity_time > REFRESH_PERIOD:
q_rnodes.append(rnode)
if rnode.num_responses == 0:
q_rnodes.append(rnode)
return q_rnodes
def on_error_received(self, node_):
pass
def on_timeout(self, node_):
if not node_.id:
return # This is a bootstrap node (just addr, no id)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a timeout from myself, WTF? Just ignore.
m_bucket = sbucket.main
rnode = m_bucket.get_rnode(node_)
if not rnode:
# This node is not in the table. Nothing to do here
return
# The node is in the table. Update it
self._update_rnode_on_timeout(rnode)
t_strikes, c_rnode = self._pinged_q_rnodes.get(node_, (None, None))
if t_strikes is None:
# The node is not being checked by a "questinable ping".
return
elif t_strikes == 0:
# This is the first timeout
self._pinged_q_rnodes[node_] = (1, c_rnode)
# Let's give it another chance
return [Query(self.ping_msg, rnode)]
elif t_strikes == 1:
# Second timeout. You're a bad node, replace if possible
# check if the candidate node is in the routing table
log_distance = self.my_node.log_distance(c_rnode)
m_bucket = self.table.get_sbucket(log_distance).main
c_rnode_in_table = m_bucket.get_rnode(c_rnode)
if c_rnode_in_table:
print 'questionable node replaced'
# replace
m_bucket.remove(rnode)
m_bucket.add(c_rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 0
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes,
exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
"""Register a query from node.
You should call this method when receiving a query from this node.
"""
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
"""Register a reply from rnode.
You should call this method when receiving a response from this rnode.
"""
rnode.rtt = rtt
current_time = time.time()
#rnode._reset_refresh_task()
if rnode.in_quarantine:
rnode.in_quarantine = rnode.last_action_ts < (current_time -
QUARANTINE_PERIOD)
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
"""Register a timeout for this rnode.
You should call this method when getting a timeout for this node.
"""
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
#Copy the bootstrap list
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._maintenance_mode = BOOTSTRAP_MODE
self._pinged_q_rnodes = {} # questionable nodes which have been
# recently pinged
self._maintenance_tasks = [self._refresh_stale_bucket,
#self._ping_a_staled_rnode,
# self._ping_a_query_received_node,
# self._ping_a_found_node,
]
def do_maintenance(self):
queries_to_send = []
maintenance_lookup_target = None
if self._maintenance_mode == BOOTSTRAP_MODE:
try:
node_ = self.bootstrap_nodes.next()
queries_to_send = [self._get_maintenance_query(node_)]
except (StopIteration):
maintenance_lookup_target = self.my_node.id
self._maintenance_mode = FIND_NODES_MODE
return (10, [], maintenance_lookup_target)
else:
maintenance_lookup_target = self._refresh_stale_bucket()
# print 'beb5', _MAINTENANCE_DELAY[self._maintenance_mode]
return (_MAINTENANCE_DELAY[self._maintenance_mode],
queries_to_send, maintenance_lookup_target)
def _refresh_stale_bucket(self):
maintenance_lookup_target = None
current_time = time.time()
for i in xrange(NUM_BUCKETS):
sbucket = self.table.get_sbucket(i)
m_bucket = sbucket.main
if not m_bucket:
continue
inactivity_time = current_time - m_bucket.last_changed_ts
if inactivity_time > REFRESH_PERIOD:
# print time.time(), '>>>>>>>>>>>>>>> refreshing bucket %d after %f secs' % (
# i, inactivity_time)
maintenance_lookup_target = self.my_node.id.generate_close_id(
i)
m_bucket.last_changed_ts = current_time
return maintenance_lookup_target
self._maintenance_mode = NORMAL_MODE
return None
def _get_maintenance_query(self, node_):
return message.OutgoingPingQuery(node_, self.my_node.id)
def on_query_received(self, node_):
'''
Return None when nothing to do
Return a list of queries when queries need to be sent (the queries
will be sent out by the caller)
'''
if self._maintenance_mode != NORMAL_MODE:
return
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a query from myself. Just ignore it.
m_bucket = sbucket.main
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: inform rnode
self._update_rnode_on_query_received(rnode)
return
# node is not in the routing table
if m_bucket.there_is_room():
# There is room in the bucket. Just add the new node.
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_query_received(rnode)
return
# No room in the main routing table
# Check whether there is a bad node to be replaced.
bad_rnode = self._pop_bad_rnode(m_bucket)
if bad_rnode:
# We have a bad node in the bucket. Replace it with the new node.
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
self.table.num_rnodes += 0
return
# No bad nodes. Check for questionable nodes
q_rnodes = self._get_questionable_rnodes(m_bucket)
queries_to_send = []
for q_rnode in q_rnodes:
# Ping questinable nodes to check whether they are still alive.
# (0 timeouts so far, candidate node)
c_rnode = node_.get_rnode(log_distance)
self._update_rnode_on_query_received(c_rnode)
self._pinged_q_rnodes[q_rnode] = [0, c_rnode]
queries_to_send.append(message.OutgoingPingQuery(node_, self.my_node.id))
return queries_to_send
def on_response_received(self, node_, rtt, nodes):
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a response from myself. Just ignore it.
m_bucket = sbucket.main
rnode = m_bucket.get_rnode(node_)
if rnode:
logger.debug('node in main')
# node in routing table: update
self._update_rnode_on_response_received(rnode, rtt)
if self._maintenance_mode == NORMAL_MODE:
m_bucket.last_changed_ts = time.time()
if node_ in self._pinged_q_rnodes:
logger.debug('remove from questionable')
rnode.questionable = False
# This node is questionable. This response proves that it is
# alive. Remove it from the questionable dict.
del self._pinged_q_rnodes[node_]
return
# The node is not in main
if m_bucket.there_is_room():
logger.debug('node not in main, there is room')
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
if self._maintenance_mode == NORMAL_MODE:
m_bucket.last_changed_ts = time.time()
return
# The main bucket is full
# if there is a bad node inside the bucket,
# replace it with the sending node_
logger.debug('node not in main, no room')
bad_rnode = self._pop_bad_rnode(m_bucket)
if bad_rnode:
logger.debug('there is a bad rnode')
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
# No need to update table
self.table.num_rnodes += 0
self._update_rnode_on_response_received(rnode, rtt)
if self._maintenance_mode == NORMAL_MODE:
m_bucket.last_changed_ts = time.time()
return
# There are no bad nodes. Ping questionable nodes (if any)
logger.debug('no bad nodes, ping questionable nodes')
q_rnodes = self._get_questionable_rnodes(m_bucket)
queries_to_send = []
for q_rnode in q_rnodes:
# (0 timeouts so far, candidate node)
c_rnode = node_.get_rnode(log_distance)
self._update_rnode_on_response_received(c_rnode, rtt)
self._pinged_q_rnodes[q_rnode] = [0, c_rnode]
queries_to_send.append(message.OutgoingPingQuery(node_,
self.my_node.id))
return queries_to_send
def _pop_bad_rnode(self, mbucket):
for rnode in mbucket.rnodes:
if rnode.timeouts_in_a_row() >= 2:
mbucket.remove(rnode)
return rnode
def _get_questionable_rnodes(self, m_bucket):
q_rnodes = []
for rnode in m_bucket.rnodes:
inactivity_time = time.time() - rnode.last_seen
if (inactivity_time > REFRESH_PERIOD
or rnode.num_responses == 0):
is_questionable = getattr(rnode, 'questionable', False)
if not is_questionable:
rnode.questionable = True
q_rnodes.append(rnode)
return q_rnodes
def on_error_received(self, node_addr):
pass
def on_timeout(self, node_):
if not node_.id:
return [] # This is a bootstrap node (just addr, no id)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return [] # Got a timeout from myself, WTF? Just ignore.
m_bucket = sbucket.main
rnode = m_bucket.get_rnode(node_)
if not rnode:
# This node is not in the table. Nothing to do here
return []
# The node is in the table. Update it
self._update_rnode_on_timeout(rnode)
t_strikes, c_rnode = self._pinged_q_rnodes.get(node_, (None, None))
if t_strikes is None:
# The node is not being checked by a "questinable ping".
return []
elif t_strikes == 0:
# This is the first timeout
self._pinged_q_rnodes[node_] = (1, c_rnode)
# Let's give it another chance
return [message.OutgoingPingQuery(node_, self.my_node.id)]
elif t_strikes == 1:
# Second timeout. You're a bad node, replace if possible
# check if the candidate node is in the routing table
log_distance = self.my_node.log_distance(c_rnode)
m_bucket = self.table.get_sbucket(log_distance).main
c_rnode_in_table = m_bucket.get_rnode(c_rnode)
if c_rnode_in_table:
print 'questionable node replaced'
# replace
m_bucket.remove(rnode)
m_bucket.add(c_rnode)
self.table.num_rnodes += 0
return []
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes,
exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
"""Register a query from node.
You should call this method when receiving a query from this node.
"""
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
"""Register a reply from rnode.
You should call this method when receiving a response from this rnode.
"""
rnode.real_rtt = rtt
current_time = time.time()
#rnode._reset_refresh_task()
if rnode.in_quarantine:
rnode.in_quarantine = \
rnode.last_action_ts < current_time - QUARANTINE_PERIOD
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
"""Register a timeout for this rnode.
You should call this method when getting a timeout for this node.
"""
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
#Copy the bootstrap list
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
self.ping_msg = message.OutgoingPingQuery(my_node.id)
self.find_closest_msg = message.OutgoingFindNodeQuery(
my_node.id, my_node.id)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [
self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
def do_maintenance(self):
queries_to_send = []
maintenance_lookup_target = None
if self._maintenance_mode == BOOTSTRAP_MODE:
try:
node_ = self.bootstrap_nodes.next()
queries_to_send = [self._get_maintenance_query(node_)]
except (StopIteration):
maintenance_lookup_target = self.my_node.id
self._maintenance_mode = NORMAL_MODE
elif self._maintenance_mode == NORMAL_MODE:
for _ in range(len(self._maintenance_tasks)):
# We try maintenance tasks till one of them actually does work
# or we have tried them all (whatever happens first) We loop
# in range because I'm going to modify self._maintenance_tasks
task = self._maintenance_tasks.pop(0)
self._maintenance_tasks.append(task)
node_ = task()
if node_:
queries_to_send = [self._get_maintenance_query(node_)]
# This task did do some work. We are done here!
break
return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send,
maintenance_lookup_target)
def _ping_a_staled_rnode(self):
# Don't have self._next_stale_maintenance_index lower than
# lowest_bucket
starting_index = self._next_stale_maintenance_index
result = None
while not result:
# Find a non-empty bucket
sbucket = self.table.get_sbucket(
self._next_stale_maintenance_index)
m_bucket = sbucket.main
self._next_stale_maintenance_index = (
self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
if m_bucket:
rnode = m_bucket.get_stalest_rnode()
if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
result = rnode
if self._next_stale_maintenance_index == starting_index:
# No node to be pinged in the whole table.
break
return result
def _ping_a_found_node(self):
num_pings = 1
if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP:
# Extra ping when bootstrapping
num_pings += 1
for _ in range(num_pings):
node_ = self._found_nodes_queue.pop(0)
if node_:
logger.debug('pinging node found: %r', node_)
return node_
return
def _ping_a_query_received_node(self):
return self._query_received_queue.pop(0)
def _ping_a_replacement_node(self):
return self._replacement_queue.pop(0)
def _get_maintenance_query(self, node_):
if not node_.id:
# Bootstrap nodes don't have id
return Query(self.find_closest_msg, node_)
if random.choice((False, True)):
# 50% chance to send find_node with my id as target
return Query(self.find_closest_msg, node_)
# 50% chance to send a find_node to fill up a non-full bucket
target_log_distance = self.table.find_next_bucket_with_room_index(
node_=node_)
if target_log_distance:
target = self.my_node.id.generate_close_id(target_log_distance)
return Query(
message.OutgoingFindNodeQuery(self.my_node.id, target), node_)
else:
# Every bucket is full. We send a ping instead.
return Query(self.ping_msg, node_)
def on_query_received(self, node_):
'''
Return None when nothing to do
Return a list of queries when queries need to be sent (the queries
will be sent out by the caller)
'''
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a query from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: inform rnode
self._update_rnode_on_query_received(rnode)
return
# node is not in the routing table
if m_bucket.there_is_room():
# There is room in the bucket: queue it
self._query_received_queue.add(node_, log_distance)
return
# No room in the main routing table
# Add to replacement table (if the bucket is not full)
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
return
def on_response_received(self, node_, rtt, nodes):
if nodes:
logger.debug('nodes found: %r', nodes)
self._found_nodes_queue.add(nodes)
logger.debug('on response received %f', rtt)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a response from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update
self._update_rnode_on_response_received(rnode, rtt)
return
# The node is not in main
rnode = r_bucket.get_rnode(node_)
if rnode:
# node in replacement table
# let's see whether there is room in the main
self._update_rnode_on_response_received(rnode, rtt)
#TODO: leave this for the maintenance task
if m_bucket.there_is_room():
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
r_bucket.remove(rnode)
return
# The node is nowhere
# Add to main table (if the bucket is not full)
#TODO: check whether in replacement_mode
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
return
# The main bucket is full
# Get the worst node in replacement bucket and see whether
# it's bad enough to be replaced by node_
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
# This node is better candidate than worst_rnode
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
return
def on_error_received(self, node_):
pass
def on_timeout(self, node_):
if not node_.id:
return # This is a bootstrap node (just addr, no id)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a timeout from myself, WTF? Just ignore.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: kick it out
self._update_rnode_on_timeout(rnode)
m_bucket.remove(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes -= 1
for r_rnode in r_bucket.sorted_by_rtt():
self._replacement_queue.add(r_rnode)
if r_bucket.there_is_room():
r_bucket.add(rnode)
else:
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode:
# Replace worst node in replacement table
r_bucket.remove(worst_rnode)
r_bucket.add(rnode)
# Node is not in main table
rnode = r_bucket.get_rnode(node_)
if rnode:
# Node in replacement table: just update rnode
self._update_rnode_on_timeout(rnode)
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes,
exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
"""Register a query from node.
You should call this method when receiving a query from this node.
"""
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
"""Register a reply from rnode.
You should call this method when receiving a response from this rnode.
"""
rnode.rtt = rtt
current_time = time.time()
#rnode._reset_refresh_task()
if rnode.in_quarantine:
rnode.in_quarantine = \
rnode.last_action_ts < current_time - QUARANTINE_PERIOD
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
"""Register a timeout for this rnode.
You should call this method when getting a timeout for this node.
"""
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes, msg_f):
self.my_node = my_node
self.bootstrapper = bootstrap.OverlayBootstrapper(
my_node.id, bootstrap_nodes, msg_f)
self.msg_f = msg_f
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [
self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
self._num_timeouts_in_a_row = 0
def _get_maintenance_lookup(self, lookup_target=None, nodes=[]):
if not lookup_target:
lookup_target = identifier.RandomId()
if not nodes:
log_distance = lookup_target.distance(self.my_node.id).log
nodes = self.get_closest_rnodes(log_distance, 0, True)
return lookup_target, nodes
def do_maintenance(self):
queries_to_send = []
maintenance_lookup = None
maintenance_delay = 0
if self._maintenance_mode == BOOTSTRAP_MODE:
(queries_to_send, maintenance_lookup,
bootstrap_delay) = self.bootstrapper.do_bootstrap(
self.table.num_rnodes)
if bootstrap_delay:
maintenance_delay = bootstrap_delay
else:
self._maintenance_mode = FILL_BUCKETS
self.bootstrapper.bootstrap_done()
elif self._maintenance_mode == FILL_BUCKETS:
if self._num_pending_filling_lookups:
self._num_pending_filling_lookups -= 1
maintenance_lookup = self._get_maintenance_lookup()
else:
self._maintenance_mode = NORMAL_MODE
elif self._maintenance_mode == NORMAL_MODE:
for _ in range(len(self._maintenance_tasks)):
# We try maintenance tasks till one of them actually does work
# or we have tried them all (whatever happens first) We loop
# in range because I'm going to modify self._maintenance_tasks
task = self._maintenance_tasks.pop(0)
self._maintenance_tasks.append(task)
node_ = task()
if node_:
queries_to_send.append(self._get_maintenance_query(node_))
# This task did do some work. We are done here!
break
if self.table.num_rnodes < MIN_RNODES:
# Ping more found nodes when routing table has few nodes
node_ = self._ping_a_found_node()
if node_:
queries_to_send.append(
self._get_maintenance_query(node_, do_fill_up=True))
if not maintenance_delay:
maintenance_delay = _MAINTENANCE_DELAY[self._maintenance_mode]
return (maintenance_delay, queries_to_send, maintenance_lookup)
def _ping_a_staled_rnode(self):
starting_index = self._next_stale_maintenance_index
result = None
while not result:
# Find a non-empty bucket
sbucket = self.table.get_sbucket(
self._next_stale_maintenance_index)
m_bucket = sbucket.main
self._next_stale_maintenance_index = (
self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
if m_bucket:
rnode = m_bucket.get_stalest_rnode()
if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
result = rnode
if self._next_stale_maintenance_index == starting_index:
# No node to be pinged in the whole table.
break
return result
def _ping_a_found_node(self):
node_ = self._found_nodes_queue.pop(0)
if node_:
logger.debug('pinging node found: %r', node_)
return node_
def _ping_a_query_received_node(self):
return self._query_received_queue.pop(0)
def _ping_a_replacement_node(self):
return self._replacement_queue.pop(0)
def _get_maintenance_query(self, node_, do_fill_up=False):
'''
if not node_.id:
# Bootstrap nodes don't have id
return message.OutgoingFindNodeQuery(node_,
self.my_node.id,
self.my_node.id, None)
'''
if do_fill_up or random.choice((False, True)):
# 50% chance to send a find_node to fill up a non-full bucket
target_log_distance = self.table.find_next_bucket_with_room_index(
node_=node_)
if target_log_distance:
target = self.my_node.id.generate_close_id(target_log_distance)
msg = self.msg_f.outgoing_find_node_query(node_, target, None)
else:
# Every bucket is full. We send a ping instead.
msg = self.msg_f.outgoing_ping_query(node_)
else:
# 50% chance to send find_node with my id as target
msg = self.msg_f.outgoing_find_node_query(node_, self.my_node.id,
None)
return msg
def on_query_received(self, node_):
'''
Return None when nothing to do
Return a list of queries when queries need to be sent (the queries
will be sent out by the caller)
'''
self._num_timeouts_in_a_row = 0
if self.bootstrapper.is_bootstrap_node(node_):
return
log_distance = self.my_node.distance(node_).log
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a query from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
if node_.ip in m_bucket.ips_in_table:
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update rnode
self._update_rnode_on_query_received(rnode)
# This IP is in the table. Stop here to avoid multiple entries
# with the same IP
return
# Now, consider adding this node to the routing table
if m_bucket.there_is_room():
# There is room in the bucket: queue it
self._query_received_queue.add(node_, log_distance)
return
# No room in the main routing table
# Add to replacement table (if the bucket is not full)
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
return
def on_response_received(self, node_, rtt, nodes):
self._num_timeouts_in_a_row = 0
if self.bootstrapper.is_bootstrap_node(node_):
return
if nodes:
logger.debug('nodes found: %r', nodes)
self._found_nodes_queue.add(nodes)
logger.debug('on response received %f', rtt)
log_distance = self.my_node.distance(node_).log
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a response from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if node_.ip in m_bucket.ips_in_table:
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update rnode
self._update_rnode_on_response_received(rnode, rtt)
# This IP is in the table. Stop here to avoid multiple entries
# with the same IP
return
# Now, consider adding this node to the routing table
rnode = r_bucket.get_rnode(node_)
if rnode:
# node in replacement table
# let's see whether there is room in the main
self._update_rnode_on_response_received(rnode, rtt)
# TODO: leave this for the maintenance task
if m_bucket.there_is_room():
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
r_bucket.remove(rnode)
return
# The node is nowhere
# Add to main table (if the bucket is not full)
# TODO: check whether in replacement_mode
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
return
# The main bucket is full
# Let's see whether this node's latency is good
current_time = time.time()
rnode_to_be_replaced = None
m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True)
for rnode in m_bucket.rnodes:
rnode_age = current_time - rnode.bucket_insertion_ts
if rtt < rnode.rtt * (1 - (rnode_age / 7200)):
# A rnode can only be replaced when the candidate node's RTT
# is shorter by a factor. Over time, this factor
# decreases. For instance, when rnode has been in the bucket
# for 30 mins (1800 secs), a candidate's RTT must be at most
# 25% of the rnode's RTT (ie. two times faster). After two
# hours, a rnode cannot be replaced by this method.
# print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % (
# rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts)
rnode_to_be_replaced = rnode
break
if rnode_to_be_replaced:
m_bucket.remove(rnode_to_be_replaced)
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
# No need to update table
self.table.num_rnodes += 0
self._update_rnode_on_response_received(rnode, rtt)
return
# Get the worst node in replacement bucket and see whether
# it's bad enough to be replaced by node_
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
# This node is better candidate than worst_rnode
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
return
def on_error_received(self, node_addr):
# if self.bootstrapper.is_bootstrap_node(node_):
# return
return
def on_timeout(self, node_):
self._num_timeouts_in_a_row += 1
if self._num_timeouts_in_a_row > MAX_TIMEOUTS_IN_A_ROW:
# stop, do not expell nodes from routing table
return []
if self.bootstrapper.is_bootstrap_node(node_):
return []
log_distance = self.my_node.distance(node_).log
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return [] # Got a timeout from myself, WTF? Just ignore.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: kick it out
self._update_rnode_on_timeout(rnode)
m_bucket.remove(rnode)
self.table.num_rnodes -= 1
for r_rnode in r_bucket.sorted_by_rtt():
self._replacement_queue.add(r_rnode)
if r_bucket.there_is_room():
r_bucket.add(rnode)
else:
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode:
# Replace worst node in replacement table
r_bucket.remove(worst_rnode)
r_bucket.add(rnode)
# Node is not in main table
rnode = r_bucket.get_rnode(node_)
if rnode:
# Node in replacement table: just update rnode
self._update_rnode_on_timeout(rnode)
return []
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes,
exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
"""Register a query from node.
You should call this method when receiving a query from this node.
"""
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
"""Register a reply from rnode.
You should call this method when receiving a response from this rnode.
"""
rnode.rtt = rtt
current_time = time.time()
# rnode._reset_refresh_task()
if rnode.in_quarantine:
rnode.in_quarantine = \
rnode.last_action_ts < current_time - QUARANTINE_PERIOD
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
"""Register a timeout for this rnode.
You should call this method when getting a timeout for this node.
"""
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
self.ping_msg = message.OutgoingPingQuery(my_node.id)
self.find_closest_msg = message.OutgoingFindNodeQuery(
my_node.id, my_node.id)
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [
self._ping_a_staled_rnode, self._ping_a_query_received_node,
self._ping_a_found_node, self._ping_a_replacement_node
]
def do_maintenance(self):
queries_to_send = []
maintenance_lookup_target = None
if self._maintenance_mode == BOOTSTRAP_MODE:
try:
node_ = self.bootstrap_nodes.next()
queries_to_send = [self._get_maintenance_query(node_)]
except StopIteration:
maintenance_lookup_target = self.my_node.id
self._maintenance_mode = NORMAL_MODE
elif self._maintenance_mode == NORMAL_MODE:
for _ in range(len(self._maintenance_tasks)):
task = self._maintenance_tasks.pop(0)
self._maintenance_tasks.append(task)
node_ = task()
if node_:
queries_to_send = [self._get_maintenance_query(node_)]
break
return (_MAINTENANCE_DELAY[self._maintenance_mode], queries_to_send,
maintenance_lookup_target)
def _ping_a_staled_rnode(self):
starting_index = self._next_stale_maintenance_index
result = None
while not result:
sbucket = self.table.get_sbucket(
self._next_stale_maintenance_index)
m_bucket = sbucket.main
self._next_stale_maintenance_index = (
self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
if m_bucket:
rnode = m_bucket.get_stalest_rnode()
if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
result = rnode
if self._next_stale_maintenance_index == starting_index:
break
return result
def _ping_a_found_node(self):
num_pings = 1
if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP:
num_pings += 1
for _ in range(num_pings):
node_ = self._found_nodes_queue.pop(0)
if node_:
logger.debug('pinging node found: %r', node_)
return node_
def _ping_a_query_received_node(self):
return self._query_received_queue.pop(0)
def _ping_a_replacement_node(self):
return self._replacement_queue.pop(0)
def _get_maintenance_query(self, node_):
if not node_.id:
return Query(self.find_closest_msg, node_)
elif random.choice((False, True)):
return Query(self.find_closest_msg, node_)
target_log_distance = self.table.find_next_bucket_with_room_index(
node_=node_)
if target_log_distance:
target = self.my_node.id.generate_close_id(target_log_distance)
return Query(
message.OutgoingFindNodeQuery(self.my_node.id, target), node_)
else:
return Query(self.ping_msg, node_)
def on_query_received(self, node_):
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except IndexError:
return
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_query_received(rnode)
return
if m_bucket.there_is_room():
self._query_received_queue.add(node_, log_distance)
return
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
def on_response_received(self, node_, rtt, nodes):
if nodes:
logger.debug('nodes found: %r', nodes)
self._found_nodes_queue.add(nodes)
logger.debug('on response received %f', rtt)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except IndexError:
return
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_response_received(rnode, rtt)
return
rnode = r_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_response_received(rnode, rtt)
if m_bucket.there_is_room():
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
r_bucket.remove(rnode)
return
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
return
current_time = time.time()
rnode_to_be_replaced = None
for rnode in reversed(m_bucket.rnodes):
rnode_age = current_time - rnode.bucket_insertion_ts
if rtt < rnode.rtt * (1 - rnode_age / 7200):
rnode_to_be_replaced = rnode
break
if rnode_to_be_replaced:
m_bucket.remove(rnode_to_be_replaced)
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.num_rnodes += 0
self._update_rnode_on_response_received(rnode, rtt)
return
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
def on_error_received(self, node_addr):
pass
def on_timeout(self, node_):
if not node_.id:
return
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except IndexError:
return
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_timeout(rnode)
m_bucket.remove(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes -= 1
for r_rnode in r_bucket.sorted_by_rtt():
self._replacement_queue.add(r_rnode)
if r_bucket.there_is_room():
r_bucket.add(rnode)
else:
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode:
r_bucket.remove(worst_rnode)
r_bucket.add(rnode)
rnode = r_bucket.get_rnode(node_)
if rnode:
self._update_rnode_on_timeout(rnode)
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes,
exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
rnode.rtt = rtt
current_time = time.time()
if rnode.in_quarantine:
rnode.in_quarantine = rnode.last_action_ts < current_time - QUARANTINE_PERIOD
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
#Copy the bootstrap list
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
self.ping_msg = message.OutgoingPingQuery(my_node.id)
self.find_closest_msg = message.OutgoingFindNodeQuery(
my_node.id,
my_node.id)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
def do_maintenance(self):
queries_to_send = []
maintenance_lookup_target = None
if self._maintenance_mode == BOOTSTRAP_MODE:
try:
node_ = self.bootstrap_nodes.next()
queries_to_send = [self._get_maintenance_query(node_)]
except (StopIteration):
maintenance_lookup_target = self.my_node.id
self._maintenance_mode = NORMAL_MODE
elif self._maintenance_mode == NORMAL_MODE:
for _ in range(len(self._maintenance_tasks)):
# We try maintenance tasks till one of them actually does work
# or we have tried them all (whatever happens first) We loop
# in range because I'm going to modify self._maintenance_tasks
task = self._maintenance_tasks.pop(0)
self._maintenance_tasks.append(task)
node_ = task()
if node_:
queries_to_send = [self._get_maintenance_query(node_)]
# This task did do some work. We are done here!
break
return (_MAINTENANCE_DELAY[self._maintenance_mode],
queries_to_send, maintenance_lookup_target)
def _ping_a_staled_rnode(self):
# Don't have self._next_stale_maintenance_index lower than
# lowest_bucket
starting_index = self._next_stale_maintenance_index
result = None
while not result:
# Find a non-empty bucket
sbucket = self.table.get_sbucket(
self._next_stale_maintenance_index)
m_bucket = sbucket.main
self._next_stale_maintenance_index = (
self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
if m_bucket:
rnode = m_bucket.get_stalest_rnode()
if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
result = rnode
if self._next_stale_maintenance_index == starting_index:
# No node to be pinged in the whole table.
break
return result
def _ping_a_found_node(self):
num_pings = 1
if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP:
# Extra ping when bootstrapping
num_pings += 1
for _ in range(num_pings):
node_ = self._found_nodes_queue.pop(0)
if node_:
logger.debug('pinging node found: %r', node_)
return node_
return
def _ping_a_query_received_node(self):
return self._query_received_queue.pop(0)
def _ping_a_replacement_node(self):
return self._replacement_queue.pop(0)
def _get_maintenance_query(self, node_):
if not node_.id:
# Bootstrap nodes don't have id
return Query(self.find_closest_msg, node_)
if random.choice((False, True)):
# 50% chance to send find_node with my id as target
return Query(self.find_closest_msg, node_)
# 50% chance to send a find_node to fill up a non-full bucket
target_log_distance = self.table.find_next_bucket_with_room_index(
node_=node_)
if target_log_distance:
target = self.my_node.id.generate_close_id(target_log_distance)
return Query(
message.OutgoingFindNodeQuery(self.my_node.id, target),
node_)
else:
# Every bucket is full. We send a ping instead.
return Query(self.ping_msg, node_)
def on_query_received(self, node_):
'''
Return None when nothing to do
Return a list of queries when queries need to be sent (the queries
will be sent out by the caller)
'''
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a query from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: inform rnode
self._update_rnode_on_query_received(rnode)
return
# node is not in the routing table
if m_bucket.there_is_room():
# There is room in the bucket: queue it
self._query_received_queue.add(node_, log_distance)
return
# No room in the main routing table
# Add to replacement table (if the bucket is not full)
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
return
def on_response_received(self, node_, rtt, nodes):
if nodes:
logger.debug('nodes found: %r', nodes)
self._found_nodes_queue.add(nodes)
logger.debug('on response received %f', rtt)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a response from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update
self._update_rnode_on_response_received(rnode, rtt)
return
# The node is not in main
rnode = r_bucket.get_rnode(node_)
if rnode:
# node in replacement table
# let's see whether there is room in the main
self._update_rnode_on_response_received(rnode, rtt)
#TODO: leave this for the maintenance task
if m_bucket.there_is_room():
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
r_bucket.remove(rnode)
return
# The node is nowhere
# Add to main table (if the bucket is not full)
#TODO: check whether in replacement_mode
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.update_lowest_index(log_distance)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
return
# The main bucket is full
# Let's see whether this node's latency is good
current_time = time.time()
rnode_to_be_replaced = None
for rnode in reversed(m_bucket.rnodes):
rnode_age = current_time - rnode.bucket_insertion_ts
if rtt < rnode.rtt * (1 - (rnode_age / 7200)):
# A rnode can only be replaced when the candidate node's RTT
# is shorter by a factor. Over time, this factor
# decreases. For instance, when rnode has been in the bucket
# for 30 mins (1800 secs), a candidate's RTT must be at most
# 50% of the rnode's RTT (ie. two times faster). After two
# hours, a rnode cannot be replaced becouse of better RTT.
# print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % (
rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts)
rnode_to_be_replaced = rnode
break
if rnode_to_be_replaced:
m_bucket.remove(rnode_to_be_replaced)
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
# No need to update table
self.table.num_rnodes += 0
self._update_rnode_on_response_received(rnode, rtt)
return
# Get the worst node in replacement bucket and see whether
# it's bad enough to be replaced by node_
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
# This node is better candidate than worst_rnode
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
return
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes):
self.my_node = my_node
#Copy the bootstrap list
self.bootstrap_nodes = iter(bootstrap_nodes)
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
def do_maintenance(self):
queries_to_send = []
maintenance_lookup_target = None
if self._maintenance_mode == BOOTSTRAP_MODE:
try:
node_ = self.bootstrap_nodes.next()
queries_to_send = [self._get_maintenance_query(node_)]
except (StopIteration):
maintenance_lookup_target = self.my_node.id
self._maintenance_mode = FILL_BUCKETS
elif self._maintenance_mode == FILL_BUCKETS:
if self._num_pending_filling_lookups:
self._num_pending_filling_lookups -= 1
maintenance_lookup_target = identifier.RandomId()
else:
self._maintenance_mode = NORMAL_MODE
elif self._maintenance_mode == NORMAL_MODE:
for _ in range(len(self._maintenance_tasks)):
# We try maintenance tasks till one of them actually does work
# or we have tried them all (whatever happens first) We loop
# in range because I'm going to modify self._maintenance_tasks
task = self._maintenance_tasks.pop(0)
self._maintenance_tasks.append(task)
node_ = task()
if node_:
queries_to_send = [self._get_maintenance_query(node_)]
# This task did do some work. We are done here!
break
return (_MAINTENANCE_DELAY[self._maintenance_mode],
queries_to_send, maintenance_lookup_target)
def _ping_a_staled_rnode(self):
starting_index = self._next_stale_maintenance_index
result = None
while not result:
# Find a non-empty bucket
sbucket = self.table.get_sbucket(
self._next_stale_maintenance_index)
m_bucket = sbucket.main
self._next_stale_maintenance_index = (
self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
if m_bucket:
rnode = m_bucket.get_stalest_rnode()
if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
result = rnode
if self._next_stale_maintenance_index == starting_index:
# No node to be pinged in the whole table.
break
return result
def _ping_a_found_node(self):
num_pings = 1
if self.table.num_rnodes < MIN_RNODES_BOOTSTRAP:
# Extra ping when bootstrapping
num_pings += 1
for _ in range(num_pings):
node_ = self._found_nodes_queue.pop(0)
if node_:
logger.debug('pinging node found: %r', node_)
return node_
return
def _ping_a_query_received_node(self):
return self._query_received_queue.pop(0)
def _ping_a_replacement_node(self):
return self._replacement_queue.pop(0)
def _get_maintenance_query(self, node_):
if not node_.id:
# Bootstrap nodes don't have id
return message.OutgoingFindNodeQuery(node_,
self.my_node.id,
self.my_node.id, None)
if random.choice((False, True)):
# 50% chance to send find_node with my id as target
return message.OutgoingFindNodeQuery(node_,
self.my_node.id,
self.my_node.id, None)
# 50% chance to send a find_node to fill up a non-full bucket
target_log_distance = self.table.find_next_bucket_with_room_index(
node_=node_)
if target_log_distance:
target = self.my_node.id.generate_close_id(target_log_distance)
return message.OutgoingFindNodeQuery(node_, self.my_node.id,
target, None)
else:
# Every bucket is full. We send a ping instead.
return message.OutgoingPingQuery(node_, self.my_node.id)
def on_query_received(self, node_):
'''
Return None when nothing to do
Return a list of queries when queries need to be sent (the queries
will be sent out by the caller)
'''
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a query from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: inform rnode
self._update_rnode_on_query_received(rnode)
return
# node is not in the routing table
if m_bucket.there_is_room():
# There is room in the bucket: queue it
self._query_received_queue.add(node_, log_distance)
return
# No room in the main routing table
# Add to replacement table (if the bucket is not full)
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
return
def on_response_received(self, node_, rtt, nodes):
if nodes:
logger.debug('nodes found: %r', nodes)
self._found_nodes_queue.add(nodes)
logger.debug('on response received %f', rtt)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a response from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update
self._update_rnode_on_response_received(rnode, rtt)
return
# The node is not in main
rnode = r_bucket.get_rnode(node_)
if rnode:
# node in replacement table
# let's see whether there is room in the main
self._update_rnode_on_response_received(rnode, rtt)
#TODO: leave this for the maintenance task
if m_bucket.there_is_room():
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
r_bucket.remove(rnode)
return
# The node is nowhere
# Add to main table (if the bucket is not full)
#TODO: check whether in replacement_mode
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
return
# The main bucket is full
# Let's see whether this node's latency is good
current_time = time.time()
rnode_to_be_replaced = None
m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True)
rnode = m_bucket.rnodes[0]
if rtt < rnode.rtt:
# Replace rnode (the node whose RTT is highest in the bucket) if
# the candidate's RTT is lower regardless of rhe rnode's age.
rnode_to_be_replaced = rnode
if rnode_to_be_replaced:
m_bucket.remove(rnode_to_be_replaced)
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
# No need to update table
self.table.num_rnodes += 0
self._update_rnode_on_response_received(rnode, rtt)
return
# Get the worst node in replacement bucket and see whether
# it's bad enough to be replaced by node_
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
# This node is better candidate than worst_rnode
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
return
def on_error_received(self, node_addr):
pass
def on_timeout(self, node_):
if not node_.id:
return [] # This is a bootstrap node (just addr, no id)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return [] # Got a timeout from myself, WTF? Just ignore.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: kick it out
self._update_rnode_on_timeout(rnode)
m_bucket.remove(rnode)
self.table.num_rnodes -= 1
for r_rnode in r_bucket.sorted_by_rtt():
self._replacement_queue.add(r_rnode)
if r_bucket.there_is_room():
r_bucket.add(rnode)
else:
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode:
# Replace worst node in replacement table
r_bucket.remove(worst_rnode)
r_bucket.add(rnode)
# Node is not in main table
rnode = r_bucket.get_rnode(node_)
if rnode:
# Node in replacement table: just update rnode
self._update_rnode_on_timeout(rnode)
return []
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes,
exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
"""Register a query from node.
You should call this method when receiving a query from this node.
"""
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
"""Register a reply from rnode.
You should call this method when receiving a response from this rnode.
"""
rnode.rtt = rtt
current_time = time.time()
#rnode._reset_refresh_task()
if rnode.in_quarantine:
rnode.in_quarantine = \
rnode.last_action_ts < current_time - QUARANTINE_PERIOD
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
"""Register a timeout for this rnode.
You should call this method when getting a timeout for this node.
"""
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes, msg_f):
self.my_node = my_node
self.bootstrapper = bootstrap.OverlayBootstrapper(my_node.id,
bootstrap_nodes, msg_f)
self.msg_f = msg_f
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
def _get_maintenance_lookup(self, lookup_target=None, nodes=[]):
if not lookup_target:
lookup_target = identifier.RandomId()
if not nodes:
log_distance = lookup_target.log_distance(self.my_node.id)
nodes = self.get_closest_rnodes(log_distance, 0, True)
return lookup_target, nodes
def do_maintenance(self):
queries_to_send = []
maintenance_lookup = None
maintenance_delay = 0
if self._maintenance_mode == BOOTSTRAP_MODE:
(queries_to_send,
maintenance_lookup,
bootstrap_delay) = self.bootstrapper.do_bootstrap(
self.table.num_rnodes)
if bootstrap_delay:
maintenance_delay = bootstrap_delay
else:
self._maintenance_mode = FILL_BUCKETS
elif self._maintenance_mode == FILL_BUCKETS:
if self._num_pending_filling_lookups:
self._num_pending_filling_lookups -= 1
maintenance_lookup = self._get_maintenance_lookup()
else:
self._maintenance_mode = NORMAL_MODE
elif self._maintenance_mode == NORMAL_MODE:
for _ in range(len(self._maintenance_tasks)):
# We try maintenance tasks till one of them actually does work
# or we have tried them all (whatever happens first) We loop
# in range because I'm going to modify self._maintenance_tasks
task = self._maintenance_tasks.pop(0)
self._maintenance_tasks.append(task)
node_ = task()
if node_:
queries_to_send.append(self._get_maintenance_query(node_))
# This task did do some work. We are done here!
break
if self.table.num_rnodes < MIN_RNODES:
# Ping more found nodes when routing table has few nodes
node_ = self._ping_a_found_node()
if node_:
queries_to_send.append(self._get_maintenance_query(
node_, do_fill_up=True))
if not maintenance_delay:
maintenance_delay = _MAINTENANCE_DELAY[self._maintenance_mode]
return (maintenance_delay, queries_to_send, maintenance_lookup)
def _ping_a_staled_rnode(self):
starting_index = self._next_stale_maintenance_index
result = None
while not result:
# Find a non-empty bucket
sbucket = self.table.get_sbucket(
self._next_stale_maintenance_index)
m_bucket = sbucket.main
self._next_stale_maintenance_index = (
self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
if m_bucket:
rnode = m_bucket.get_stalest_rnode()
if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
result = rnode
if self._next_stale_maintenance_index == starting_index:
# No node to be pinged in the whole table.
break
return result
def _ping_a_found_node(self):
node_ = self._found_nodes_queue.pop(0)
if node_:
logger.debug('pinging node found: %r', node_)
return node_
def _ping_a_query_received_node(self):
return self._query_received_queue.pop(0)
def _ping_a_replacement_node(self):
return self._replacement_queue.pop(0)
def _get_maintenance_query(self, node_, do_fill_up=False):
'''
if not node_.id:
# Bootstrap nodes don't have id
return message.OutgoingFindNodeQuery(node_,
self.my_node.id,
self.my_node.id, None)
'''
if do_fill_up or random.choice((False, True)):
# 50% chance to send a find_node to fill up a non-full bucket
target_log_distance = self.table.find_next_bucket_with_room_index(
node_=node_)
if target_log_distance:
target = self.my_node.id.generate_close_id(target_log_distance)
msg = self.msg_f.outgoing_find_node_query(node_,
target, None)
else:
# Every bucket is full. We send a ping instead.
msg = self.msg_f.outgoing_ping_query(node_)
else:
# 50% chance to send find_node with my id as target
msg = self.msg_f.outgoing_find_node_query(node_,
self.my_node.id, None)
return msg
def on_query_received(self, node_):
'''
Return None when nothing to do
Return a list of queries when queries need to be sent (the queries
will be sent out by the caller)
'''
if self.bootstrapper.is_bootstrap_node(node_):
return
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a query from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: inform rnode
self._update_rnode_on_query_received(rnode)
return
# node is not in the routing table
if m_bucket.there_is_room():
# There is room in the bucket: queue it
self._query_received_queue.add(node_, log_distance)
return
# No room in the main routing table
# Add to replacement table (if the bucket is not full)
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
return
def on_response_received(self, node_, rtt, nodes):
if self.bootstrapper.is_bootstrap_node(node_):
return
if nodes:
logger.debug('nodes found: %r', nodes)
self._found_nodes_queue.add(nodes)
logger.debug('on response received %f', rtt)
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a response from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update
self._update_rnode_on_response_received(rnode, rtt)
return
# The node is not in main
rnode = r_bucket.get_rnode(node_)
if rnode:
# node in replacement table
# let's see whether there is room in the main
self._update_rnode_on_response_received(rnode, rtt)
#TODO: leave this for the maintenance task
if m_bucket.there_is_room():
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
r_bucket.remove(rnode)
return
# The node is nowhere
# Add to main table (if the bucket is not full)
#TODO: check whether in replacement_mode
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
return
# The main bucket is full
# Let's see whether this node's latency is good
current_time = time.time()
rnode_to_be_replaced = None
m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True)
for rnode in m_bucket.rnodes:
rnode_age = current_time - rnode.bucket_insertion_ts
if rtt < rnode.rtt * (1 - (rnode_age / 7200)):
# A rnode can only be replaced when the candidate node's RTT
# is shorter by a factor. Over time, this factor
# decreases. For instance, when rnode has been in the bucket
# for 30 mins (1800 secs), a candidate's RTT must be at most
# 25% of the rnode's RTT (ie. two times faster). After two
# hours, a rnode cannot be replaced by this method.
# print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % (
# rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts)
rnode_to_be_replaced = rnode
break
if rnode_to_be_replaced:
m_bucket.remove(rnode_to_be_replaced)
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
# No need to update table
self.table.num_rnodes += 0
self._update_rnode_on_response_received(rnode, rtt)
return
# Get the worst node in replacement bucket and see whether
# it's bad enough to be replaced by node_
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
# This node is better candidate than worst_rnode
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
return
def on_error_received(self, node_addr):
if self.bootstrapper.is_bootstrap_node(node_):
return
return
def on_timeout(self, node_):
if self.bootstrapper.is_bootstrap_node(node_):
return
log_distance = self.my_node.log_distance(node_)
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return [] # Got a timeout from myself, WTF? Just ignore.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: kick it out
self._update_rnode_on_timeout(rnode)
m_bucket.remove(rnode)
self.table.num_rnodes -= 1
for r_rnode in r_bucket.sorted_by_rtt():
self._replacement_queue.add(r_rnode)
if r_bucket.there_is_room():
r_bucket.add(rnode)
else:
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode:
# Replace worst node in replacement table
r_bucket.remove(worst_rnode)
r_bucket.add(rnode)
# Node is not in main table
rnode = r_bucket.get_rnode(node_)
if rnode:
# Node in replacement table: just update rnode
self._update_rnode_on_timeout(rnode)
return []
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes,
exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
"""Register a query from node.
You should call this method when receiving a query from this node.
"""
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
"""Register a reply from rnode.
You should call this method when receiving a response from this rnode.
"""
rnode.rtt = rtt
current_time = time.time()
#rnode._reset_refresh_task()
if rnode.in_quarantine:
rnode.in_quarantine = \
rnode.last_action_ts < current_time - QUARANTINE_PERIOD
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
"""Register a timeout for this rnode.
You should call this method when getting a timeout for this node.
"""
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far