class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes, msg_f):
self.my_node = my_node
self.bootstrapper = bootstrap.OverlayBootstrapper(my_node.id,
bootstrap_nodes, msg_f)
self.msg_f = msg_f
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
def _get_maintenance_lookup(self, lookup_target=None, nodes=[]):
if not lookup_target:
lookup_target = identifier.RandomId()
if not nodes:
log_distance = lookup_target.distance(self.my_node.id).log
nodes = self.get_closest_rnodes(log_distance, 0, True)
return lookup_target, nodes
def do_maintenance(self):
queries_to_send = []
maintenance_lookup = None
maintenance_delay = 0
if self._maintenance_mode == BOOTSTRAP_MODE:
(queries_to_send,
maintenance_lookup,
bootstrap_delay) = self.bootstrapper.do_bootstrap(
self.table.num_rnodes)
if bootstrap_delay:
maintenance_delay = bootstrap_delay
else:
self._maintenance_mode = FILL_BUCKETS
elif self._maintenance_mode == FILL_BUCKETS:
if self._num_pending_filling_lookups:
self._num_pending_filling_lookups -= 1
maintenance_lookup = self._get_maintenance_lookup()
else:
self._maintenance_mode = NORMAL_MODE
elif self._maintenance_mode == NORMAL_MODE:
for _ in range(len(self._maintenance_tasks)):
# We try maintenance tasks till one of them actually does work
# or we have tried them all (whatever happens first) We loop
# in range because I'm going to modify self._maintenance_tasks
task = self._maintenance_tasks.pop(0)
self._maintenance_tasks.append(task)
node_ = task()
if node_:
queries_to_send.append(self._get_maintenance_query(node_))
# This task did do some work. We are done here!
break
if self.table.num_rnodes < MIN_RNODES:
# Ping more found nodes when routing table has few nodes
node_ = self._ping_a_found_node()
if node_:
queries_to_send.append(self._get_maintenance_query(
node_, do_fill_up=True))
if not maintenance_delay:
maintenance_delay = _MAINTENANCE_DELAY[self._maintenance_mode]
return (maintenance_delay, queries_to_send, maintenance_lookup)
def _ping_a_staled_rnode(self):
starting_index = self._next_stale_maintenance_index
result = None
while not result:
# Find a non-empty bucket
sbucket = self.table.get_sbucket(
self._next_stale_maintenance_index)
m_bucket = sbucket.main
self._next_stale_maintenance_index = (
self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
if m_bucket:
rnode = m_bucket.get_stalest_rnode()
if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
result = rnode
if self._next_stale_maintenance_index == starting_index:
# No node to be pinged in the whole table.
break
return result
def _ping_a_found_node(self):
node_ = self._found_nodes_queue.pop(0)
if node_:
logger.debug('pinging node found: %r', node_)
return node_
def _ping_a_query_received_node(self):
return self._query_received_queue.pop(0)
def _ping_a_replacement_node(self):
return self._replacement_queue.pop(0)
def _get_maintenance_query(self, node_, do_fill_up=False):
'''
if not node_.id:
# Bootstrap nodes don't have id
return message.OutgoingFindNodeQuery(node_,
self.my_node.id,
self.my_node.id, None)
'''
if do_fill_up or random.choice((False, True)):
# 50% chance to send a find_node to fill up a non-full bucket
target_log_distance = self.table.find_next_bucket_with_room_index(
node_=node_)
if target_log_distance:
target = self.my_node.id.generate_close_id(target_log_distance)
msg = self.msg_f.outgoing_find_node_query(node_,
target, None)
else:
# Every bucket is full. We send a ping instead.
msg = self.msg_f.outgoing_ping_query(node_)
else:
# 50% chance to send find_node with my id as target
msg = self.msg_f.outgoing_find_node_query(node_,
self.my_node.id, None)
return msg
def on_query_received(self, node_):
'''
Return None when nothing to do
Return a list of queries when queries need to be sent (the queries
will be sent out by the caller)
'''
if self.bootstrapper.is_bootstrap_node(node_):
return
log_distance = self.my_node.distance(node_).log
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a query from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
if node_.ip in m_bucket.ips_in_table:
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update rnode
self._update_rnode_on_query_received(rnode)
# This IP is in the table. Stop here to avoid multiple entries
# with the same IP
return
# Now, consider adding this node to the routing table
if m_bucket.there_is_room():
# There is room in the bucket: queue it
self._query_received_queue.add(node_, log_distance)
return
# No room in the main routing table
# Add to replacement table (if the bucket is not full)
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
return
def on_response_received(self, node_, rtt, nodes):
if self.bootstrapper.is_bootstrap_node(node_):
return
if nodes:
logger.debug('nodes found: %r', nodes)
self._found_nodes_queue.add(nodes)
logger.debug('on response received %f', rtt)
log_distance = self.my_node.distance(node_).log
try:
sbucket = self.table.get_sbucket(log_distance)
except(IndexError):
return # Got a response from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if node_.ip in m_bucket.ips_in_table:
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update rnode
self._update_rnode_on_response_received(rnode, rtt)
# This IP is in the table. Stop here to avoid multiple entries
# with the same IP
return
# Now, consider adding this node to the routing table
rnode = r_bucket.get_rnode(node_)
if rnode:
# node in replacement table
# let's see whether there is room in the main
self._update_rnode_on_response_received(rnode, rtt)
#TODO: leave this for the maintenance task
if m_bucket.there_is_room():
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
r_bucket.remove(rnode)
return
# The node is nowhere
# Add to main table (if the bucket is not full)
#TODO: check whether in replacement_mode
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
return
# The main bucket is full
# Let's see whether this node's latency is good
current_time = time.time()
rnode_to_be_replaced = None
m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True)
for rnode in m_bucket.rnodes:
rnode_age = current_time - rnode.bucket_insertion_ts
if rtt < rnode.rtt * (1 - (rnode_age / 7200)):
# A rnode can only be replaced when the candidate node's RTT
# is shorter by a factor. Over time, this factor
# decreases. For instance, when rnode has been in the bucket
# for 30 mins (1800 secs), a candidate's RTT must be at most
# 25% of the rnode's RTT (ie. two times faster). After two
# hours, a rnode cannot be replaced by this method.
# print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % (
# rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts)
rnode_to_be_replaced = rnode
break
if rnode_to_be_replaced:
m_bucket.remove(rnode_to_be_replaced)
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
# No need to update table
self.table.num_rnodes += 0
self._update_rnode_on_response_received(rnode, rtt)
return
# Get the worst node in replacement bucket and see whether
# it's bad enough to be replaced by node_
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
# This node is better candidate than worst_rnode
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
return
def on_error_received(self, node_addr):
# if self.bootstrapper.is_bootstrap_node(node_):
# return
return
def on_timeout(self, node_):
if self.bootstrapper.is_bootstrap_node(node_):
return
log_distance = self.my_node.distance(node_).log
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return [] # Got a timeout from myself, WTF? Just ignore.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: kick it out
self._update_rnode_on_timeout(rnode)
m_bucket.remove(rnode)
self.table.num_rnodes -= 1
for r_rnode in r_bucket.sorted_by_rtt():
self._replacement_queue.add(r_rnode)
if r_bucket.there_is_room():
r_bucket.add(rnode)
else:
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode:
# Replace worst node in replacement table
r_bucket.remove(worst_rnode)
r_bucket.add(rnode)
# Node is not in main table
rnode = r_bucket.get_rnode(node_)
if rnode:
# Node in replacement table: just update rnode
self._update_rnode_on_timeout(rnode)
return []
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes,
exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
"""Register a query from node.
You should call this method when receiving a query from this node.
"""
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
"""Register a reply from rnode.
You should call this method when receiving a response from this rnode.
"""
rnode.rtt = rtt
current_time = time.time()
#rnode._reset_refresh_task()
if rnode.in_quarantine:
rnode.in_quarantine = \
rnode.last_action_ts < current_time - QUARANTINE_PERIOD
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
"""Register a timeout for this rnode.
You should call this method when getting a timeout for this node.
"""
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, querier, bootstrap_nodes):
self.my_node = my_node
self.querier = querier
#Copy the bootstrap list
self.bootstrap_nodes = [n for n in bootstrap_nodes]
self.main = RoutingTable(my_node, NODES_PER_BUCKET)
self.replacement = RoutingTable(my_node, NODES_PER_BUCKET)
self.ping_msg = message.OutgoingPingQuery(my_node.id)
self.find_node_msg = message.OutgoingFindNodeQuery(
my_node.id, my_node.id)
self.mode = BOOTSTRAP_MODE
self.num_concurrent_refresh_msgs = 0
#This must be called by an external party: self.do_bootstrap()
#After initializing callbacks
# Add myself to the routing table
rnode = self.main.add(my_node)
self._reset_refresh_task(rnode)
def do_bootstrap(self):
if self.main.num_rnodes > MIN_RNODES_BOOTSTRAP:
# Enough nodes. Stop bootstrap.
return
for _ in xrange(NUM_NODES_PER_BOOTSTRAP_STEP):
if not self.bootstrap_nodes:
self.mode = NORMAL_MODE
return
index = random.randint(0, len(self.bootstrap_nodes) - 1)
self.querier.send_query(self.find_node_msg,
self.bootstrap_nodes[index], None, None,
None)
del self.bootstrap_nodes[index]
#TODO2: Don't use querier's rpc_m
self.querier.rpc_m.call_later(BOOTSTRAP_DELAY, self.do_bootstrap)
def on_query_received(self, node_):
try:
rnode = self.main.get_rnode(node_)
except RnodeNotFound:
pass # node is not in the main table
else:
# node in routing table: inform rnode
rnode.on_query_received()
self._reset_refresh_task(rnode)
return
# Node is not in routing table
# Check reachability (if the bucket is not full)
if self.main.there_is_room(node_):
# there is room in the bucket: ping node to check reachability
self._refresh_now(node_)
return
# No room in the main routing table
# Add to replacement table (if the bucket is not full)
bucket = self.replacement.get_bucket(node_)
worst_rnode = self._worst_rnode(bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
self.replacement.remove(worst_rnode)
self.replacement.add(node_)
def on_response_received(self, node_): #TODO2:, rtt=0):
try:
rnode = self.main.get_rnode(node_)
except (RnodeNotFound):
pass
else:
# node in routing table: refresh it
rnode.on_response_received()
self._reset_refresh_task(rnode)
return
# The node is not in main
try:
rnode = self.replacement.get_rnode(node_)
except (RnodeNotFound):
pass
else:
# node in replacement table
# let's see whether there is room in the main
rnode.on_response_received()
if self.main.there_is_room(node_):
rnode = self.main.add(rnode)
self._reset_refresh_task(rnode)
self.replacement.remove(rnode)
return
# The node is nowhere
# Add to replacement table (if the bucket is not full)
bucket = self.replacement.get_bucket(node_)
if self.main.there_is_room(node_):
if not bucket.rnodes:
# Replacement is empty
rnode = self.main.add(node_)
self._reset_refresh_task(rnode)
return
# The main bucket is full or the repl bucket is not empty
worst_rnode = self._worst_rnode(bucket.rnodes)
# Get the worst node in replacement bucket and see whether
# it's bad enough to be replaced by node_
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
# This node is better candidate than worst_rnode
self.replacement.remove(worst_rnode)
try:
self.replacement.add(node_)
except (BucketFullError):
pass
def on_error_received(self, node_):
pass
def on_timeout(self, node_):
if node_ is self.my_node:
raise Exception, 'I got a timeout from myself!!!'
if not node_.id:
return # This is a bootstrap node (just addr, no id)
try:
rnode = self.main.get_rnode(node_)
except RnodeNotFound:
pass
else:
# node in routing table: check whether it should be removed
rnode.on_timeout()
replacement_bucket = self.replacement.get_bucket(node_)
self._refresh_replacement_bucket(replacement_bucket)
self.main.remove(rnode)
try:
self.replacement.add(rnode)
except (BucketFullError):
worst_rnode = self._worst_rnode(replacement_bucket.rnodes)
if worst_rnode:
# Replace worst node in replacement table
self.replacement.remove(worst_rnode)
self._refresh_replacement_bucket(replacement_bucket)
# We don't want to ping the node which just did timeout
self.replacement.add(rnode)
# Node is not in main table
try:
rnode = self.replacement.get_rnode(node_)
except RnodeNotFound:
pass # the node is not in any table. Nothing to do here.
else:
# Node in replacement table: just update rnode
rnode.on_timeout()
def on_nodes_found(self, nodes):
#FIXME: this will send ping at exponential rate
#not good!!!!
logger.debug('nodes found: %r', nodes)
for node_ in nodes:
try:
rnode = self.main.get_rnode(node_)
except RnodeNotFound:
# Not in the main: ping it if there is room in main
if self.main.there_is_room(node_):
logger.debug('pinging node found: %r', node_)
self._refresh_now(node_, NO_PRIORITY)
#TODO2: prefer NS
def get_closest_rnodes(self, target_id, num_nodes=DEFAULT_NUM_NODES):
return self.main.get_closest_rnodes(target_id, num_nodes)
def get_all_rnodes(self):
return (self.main.get_all_rnodes(), self.replacement.get_all_rnodes())
def print_stats(self):
print '=== MAIN ==='
self.main.print_stats()
print '=== REPLACEMENT ==='
self.replacement.print_stats()
print '=== ==='
def _refresh_now(self, node_, priority=PRIORITY):
if priority == NO_PRIORITY and \
self.num_concurrent_refresh_msgs > MAX_CONCURRENT_REFRESH_MSGS:
return
self.num_concurrent_refresh_msgs += 1
return self.querier.send_query(self.find_node_msg, node_,
self._refresh_now_callback,
self._refresh_now_callback,
self._refresh_now_callback)
def _reset_refresh_task(self, rnode):
if rnode.refresh_task:
# Cancel the current refresh task
rnode.refresh_task.cancel()
if rnode.in_quarantine:
rnode.refresh_task = self._refresh_later(rnode, QUARANTINE_PERIOD)
else:
rnode.refresh_task = self._refresh_later(rnode)
def _refresh_later(self, rnode, delay=REFRESH_PERIOD):
return self.querier.send_query_later(delay, self.find_node_msg, rnode,
None, None, None)
def _do_nothing(self, *args, **kwargs):
pass
def _refresh_now_callback(self, *args, **kwargs):
self.num_concurrent_refresh_msgs -= 1
def _refresh_replacement_bucket(self, bucket):
for rnode in bucket.rnodes:
if rnode.is_ns:
# We give advantage to NS nodes
self._refresh_now(rnode)
else:
self._refresh_later(rnode, REFRESH_DELAY_FOR_NON_NS)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, querier, bootstrap_nodes):
self.my_node = my_node
self.querier = querier
#Copy the bootstrap list
self.bootstrap_nodes = [n for n in bootstrap_nodes]
self.main = RoutingTable(my_node, NODES_PER_BUCKET)
self.replacement = RoutingTable(my_node, NODES_PER_BUCKET)
self.ping_msg = message.OutgoingPingQuery(my_node.id)
self.find_node_msg = message.OutgoingFindNodeQuery(
my_node.id,
my_node.id)
self.mode = BOOTSTRAP_MODE
self.num_concurrent_refresh_msgs = 0
#This must be called by an external party: self.do_bootstrap()
#After initializing callbacks
# Add myself to the routing table
rnode = self.main.add(my_node)
self._reset_refresh_task(rnode)
def do_bootstrap(self):
if self.main.num_rnodes > MIN_RNODES_BOOTSTRAP:
# Enough nodes. Stop bootstrap.
return
for _ in xrange(NUM_NODES_PER_BOOTSTRAP_STEP):
if not self.bootstrap_nodes:
self.mode = NORMAL_MODE
return
index = random.randint(0,
len(self.bootstrap_nodes) - 1)
self.querier.send_query(self.find_node_msg,
self.bootstrap_nodes[index],
None,
None,
None)
del self.bootstrap_nodes[index]
#TODO2: Don't use querier's rpc_m
self.querier.rpc_m.call_later(BOOTSTRAP_DELAY,
self.do_bootstrap)
def on_query_received(self, node_):
try:
rnode = self.main.get_rnode(node_)
except RnodeNotFound:
pass # node is not in the main table
else:
# node in routing table: inform rnode
rnode.on_query_received()
self._reset_refresh_task(rnode)
return
# Node is not in routing table
# Check reachability (if the bucket is not full)
if self.main.there_is_room(node_):
# there is room in the bucket: ping node to check reachability
self._refresh_now(node_)
return
# No room in the main routing table
# Add to replacement table (if the bucket is not full)
bucket = self.replacement.get_bucket(node_)
worst_rnode = self._worst_rnode(bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
self.replacement.remove(worst_rnode)
self.replacement.add(node_)
def on_response_received(self, node_): #TODO2:, rtt=0):
try:
rnode = self.main.get_rnode(node_)
except (RnodeNotFound):
pass
else:
# node in routing table: refresh it
rnode.on_response_received()
self._reset_refresh_task(rnode)
return
# The node is not in main
try:
rnode = self.replacement.get_rnode(node_)
except (RnodeNotFound):
pass
else:
# node in replacement table
# let's see whether there is room in the main
rnode.on_response_received()
if self.main.there_is_room(node_):
rnode = self.main.add(rnode)
self._reset_refresh_task(rnode)
self.replacement.remove(rnode)
return
# The node is nowhere
# Add to replacement table (if the bucket is not full)
bucket = self.replacement.get_bucket(node_)
if self.main.there_is_room(node_):
if not bucket.rnodes:
# Replacement is empty
rnode = self.main.add(node_)
self._reset_refresh_task(rnode)
return
# The main bucket is full or the repl bucket is not empty
worst_rnode = self._worst_rnode(bucket.rnodes)
# Get the worst node in replacement bucket and see whether
# it's bad enough to be replaced by node_
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
# This node is better candidate than worst_rnode
self.replacement.remove(worst_rnode)
try:
self.replacement.add(node_)
except (BucketFullError):
pass
def on_error_received(self, node_):
pass
def on_timeout(self, node_):
if node_ is self.my_node:
raise Exception, 'I got a timeout from myself!!!'
if not node_.id:
return # This is a bootstrap node (just addr, no id)
try:
rnode = self.main.get_rnode(node_)
except RnodeNotFound:
pass
else:
# node in routing table: check whether it should be removed
rnode.on_timeout()
replacement_bucket = self.replacement.get_bucket(node_)
self._refresh_replacement_bucket(replacement_bucket)
self.main.remove(rnode)
try:
self.replacement.add(rnode)
except (BucketFullError):
worst_rnode = self._worst_rnode(replacement_bucket.rnodes)
if worst_rnode:
# Replace worst node in replacement table
self.replacement.remove(worst_rnode)
self._refresh_replacement_bucket(replacement_bucket)
# We don't want to ping the node which just did timeout
self.replacement.add(rnode)
# Node is not in main table
try:
rnode = self.replacement.get_rnode(node_)
except RnodeNotFound:
pass # the node is not in any table. Nothing to do here.
else:
# Node in replacement table: just update rnode
rnode.on_timeout()
def on_nodes_found(self, nodes):
#FIXME: this will send ping at exponential rate
#not good!!!!
logger.debug('nodes found: %r', nodes)
for node_ in nodes:
try:
rnode = self.main.get_rnode(node_)
except RnodeNotFound:
# Not in the main: ping it if there is room in main
if self.main.there_is_room(node_):
logger.debug('pinging node found: %r', node_)
self._refresh_now(node_, NO_PRIORITY)
#TODO2: prefer NS
def get_closest_rnodes(self, target_id, num_nodes=DEFAULT_NUM_NODES):
return self.main.get_closest_rnodes(target_id, num_nodes)
def get_all_rnodes(self):
return (self.main.get_all_rnodes(),
self.replacement.get_all_rnodes())
def print_stats(self):
print '=== MAIN ==='
self.main.print_stats()
print '=== REPLACEMENT ==='
self.replacement.print_stats()
print '=== ==='
def _refresh_now(self, node_, priority=PRIORITY):
if priority == NO_PRIORITY and \
self.num_concurrent_refresh_msgs > MAX_CONCURRENT_REFRESH_MSGS:
return
self.num_concurrent_refresh_msgs += 1
return self.querier.send_query(self.find_node_msg,
node_,
self._refresh_now_callback,
self._refresh_now_callback,
self._refresh_now_callback)
def _reset_refresh_task(self, rnode):
if rnode.refresh_task:
# Cancel the current refresh task
rnode.refresh_task.cancel()
if rnode.in_quarantine:
rnode.refresh_task = self._refresh_later(rnode,
QUARANTINE_PERIOD)
else:
rnode.refresh_task = self._refresh_later(rnode)
def _refresh_later(self, rnode, delay=REFRESH_PERIOD):
return self.querier.send_query_later(delay,
self.find_node_msg,
rnode,
None,
None,
None)
def _do_nothing(self, *args, **kwargs):
pass
def _refresh_now_callback(self, *args, **kwargs):
self.num_concurrent_refresh_msgs -= 1
def _refresh_replacement_bucket(self, bucket):
for rnode in bucket.rnodes:
if rnode.is_ns:
# We give advantage to NS nodes
self._refresh_now(rnode)
else:
self._refresh_later(rnode, REFRESH_DELAY_FOR_NON_NS)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
class RoutingManager(object):
def __init__(self, my_node, bootstrap_nodes, msg_f):
self.my_node = my_node
self.bootstrapper = bootstrap.OverlayBootstrapper(
my_node.id, bootstrap_nodes, msg_f)
self.msg_f = msg_f
self.table = RoutingTable(my_node, NODES_PER_BUCKET)
# maintenance variables
self._next_stale_maintenance_index = 0
self._maintenance_mode = BOOTSTRAP_MODE
self._replacement_queue = _ReplacementQueue(self.table)
self._query_received_queue = _QueryReceivedQueue(self.table)
self._found_nodes_queue = _FoundNodesQueue(self.table)
self._maintenance_tasks = [
self._ping_a_staled_rnode,
self._ping_a_query_received_node,
self._ping_a_found_node,
self._ping_a_replacement_node,
]
self._num_pending_filling_lookups = NUM_FILLING_LOOKUPS
def _get_maintenance_lookup(self, lookup_target=None, nodes=[]):
if not lookup_target:
lookup_target = identifier.RandomId()
if not nodes:
log_distance = lookup_target.distance(self.my_node.id).log
nodes = self.get_closest_rnodes(log_distance, 0, True)
return lookup_target, nodes
def do_maintenance(self):
queries_to_send = []
maintenance_lookup = None
maintenance_delay = 0
if self._maintenance_mode == BOOTSTRAP_MODE:
(queries_to_send, maintenance_lookup,
bootstrap_delay) = self.bootstrapper.do_bootstrap(
self.table.num_rnodes)
if bootstrap_delay:
maintenance_delay = bootstrap_delay
else:
self._maintenance_mode = FILL_BUCKETS
elif self._maintenance_mode == FILL_BUCKETS:
if self._num_pending_filling_lookups:
self._num_pending_filling_lookups -= 1
maintenance_lookup = self._get_maintenance_lookup()
else:
self._maintenance_mode = NORMAL_MODE
elif self._maintenance_mode == NORMAL_MODE:
for _ in range(len(self._maintenance_tasks)):
# We try maintenance tasks till one of them actually does work
# or we have tried them all (whatever happens first) We loop
# in range because I'm going to modify self._maintenance_tasks
task = self._maintenance_tasks.pop(0)
self._maintenance_tasks.append(task)
node_ = task()
if node_:
queries_to_send.append(self._get_maintenance_query(node_))
# This task did do some work. We are done here!
break
if self.table.num_rnodes < MIN_RNODES:
# Ping more found nodes when routing table has few nodes
node_ = self._ping_a_found_node()
if node_:
queries_to_send.append(
self._get_maintenance_query(node_, do_fill_up=True))
if not maintenance_delay:
maintenance_delay = _MAINTENANCE_DELAY[self._maintenance_mode]
return (maintenance_delay, queries_to_send, maintenance_lookup)
def _ping_a_staled_rnode(self):
starting_index = self._next_stale_maintenance_index
result = None
while not result:
# Find a non-empty bucket
sbucket = self.table.get_sbucket(
self._next_stale_maintenance_index)
m_bucket = sbucket.main
self._next_stale_maintenance_index = (
self._next_stale_maintenance_index + 1) % (NUM_BUCKETS - 1)
if m_bucket:
rnode = m_bucket.get_stalest_rnode()
if time.time() > rnode.last_seen + QUARANTINE_PERIOD:
result = rnode
if self._next_stale_maintenance_index == starting_index:
# No node to be pinged in the whole table.
break
return result
def _ping_a_found_node(self):
node_ = self._found_nodes_queue.pop(0)
if node_:
logger.debug('pinging node found: %r', node_)
return node_
def _ping_a_query_received_node(self):
return self._query_received_queue.pop(0)
def _ping_a_replacement_node(self):
return self._replacement_queue.pop(0)
def _get_maintenance_query(self, node_, do_fill_up=False):
'''
if not node_.id:
# Bootstrap nodes don't have id
return message.OutgoingFindNodeQuery(node_,
self.my_node.id,
self.my_node.id, None)
'''
if do_fill_up or random.choice((False, True)):
# 50% chance to send a find_node to fill up a non-full bucket
target_log_distance = self.table.find_next_bucket_with_room_index(
node_=node_)
if target_log_distance:
target = self.my_node.id.generate_close_id(target_log_distance)
msg = self.msg_f.outgoing_find_node_query(node_, target, None)
else:
# Every bucket is full. We send a ping instead.
msg = self.msg_f.outgoing_ping_query(node_)
else:
# 50% chance to send find_node with my id as target
msg = self.msg_f.outgoing_find_node_query(node_, self.my_node.id,
None)
return msg
def on_query_received(self, node_):
'''
Return None when nothing to do
Return a list of queries when queries need to be sent (the queries
will be sent out by the caller)
'''
if self.bootstrapper.is_bootstrap_node(node_):
return
log_distance = self.my_node.distance(node_).log
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a query from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
if node_.ip in m_bucket.ips_in_table:
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update rnode
self._update_rnode_on_query_received(rnode)
# This IP is in the table. Stop here to avoid multiple entries
# with the same IP
return
# Now, consider adding this node to the routing table
if m_bucket.there_is_room():
# There is room in the bucket: queue it
self._query_received_queue.add(node_, log_distance)
return
# No room in the main routing table
# Add to replacement table (if the bucket is not full)
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_query_received(rnode)
return
def on_response_received(self, node_, rtt, nodes):
if self.bootstrapper.is_bootstrap_node(node_):
return
if nodes:
logger.debug('nodes found: %r', nodes)
self._found_nodes_queue.add(nodes)
logger.debug('on response received %f', rtt)
log_distance = self.my_node.distance(node_).log
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return # Got a response from myself. Just ignore it.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if node_.ip in m_bucket.ips_in_table:
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: update rnode
self._update_rnode_on_response_received(rnode, rtt)
# This IP is in the table. Stop here to avoid multiple entries
# with the same IP
return
# Now, consider adding this node to the routing table
rnode = r_bucket.get_rnode(node_)
if rnode:
# node in replacement table
# let's see whether there is room in the main
self._update_rnode_on_response_received(rnode, rtt)
#TODO: leave this for the maintenance task
if m_bucket.there_is_room():
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
r_bucket.remove(rnode)
return
# The node is nowhere
# Add to main table (if the bucket is not full)
#TODO: check whether in replacement_mode
if m_bucket.there_is_room():
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
self.table.num_rnodes += 1
self._update_rnode_on_response_received(rnode, rtt)
return
# The main bucket is full
# Let's see whether this node's latency is good
current_time = time.time()
rnode_to_be_replaced = None
m_bucket.rnodes.sort(key=attrgetter('rtt'), reverse=True)
for rnode in m_bucket.rnodes:
rnode_age = current_time - rnode.bucket_insertion_ts
if rtt < rnode.rtt * (1 - (rnode_age / 7200)):
# A rnode can only be replaced when the candidate node's RTT
# is shorter by a factor. Over time, this factor
# decreases. For instance, when rnode has been in the bucket
# for 30 mins (1800 secs), a candidate's RTT must be at most
# 25% of the rnode's RTT (ie. two times faster). After two
# hours, a rnode cannot be replaced by this method.
# print 'RTT replacement: newRTT: %f, oldRTT: %f, age: %f' % (
# rtt, rnode.rtt, current_time - rnode.bucket_insertion_ts)
rnode_to_be_replaced = rnode
break
if rnode_to_be_replaced:
m_bucket.remove(rnode_to_be_replaced)
rnode = node_.get_rnode(log_distance)
m_bucket.add(rnode)
# No need to update table
self.table.num_rnodes += 0
self._update_rnode_on_response_received(rnode, rtt)
return
# Get the worst node in replacement bucket and see whether
# it's bad enough to be replaced by node_
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode \
and worst_rnode.timeouts_in_a_row() > MAX_NUM_TIMEOUTS:
# This node is better candidate than worst_rnode
r_bucket.remove(worst_rnode)
rnode = node_.get_rnode(log_distance)
r_bucket.add(rnode)
self._update_rnode_on_response_received(rnode, rtt)
return
def on_error_received(self, node_addr):
# if self.bootstrapper.is_bootstrap_node(node_):
# return
return
def on_timeout(self, node_):
if self.bootstrapper.is_bootstrap_node(node_):
return
log_distance = self.my_node.distance(node_).log
try:
sbucket = self.table.get_sbucket(log_distance)
except (IndexError):
return [] # Got a timeout from myself, WTF? Just ignore.
m_bucket = sbucket.main
r_bucket = sbucket.replacement
rnode = m_bucket.get_rnode(node_)
if rnode:
# node in routing table: kick it out
self._update_rnode_on_timeout(rnode)
m_bucket.remove(rnode)
self.table.num_rnodes -= 1
for r_rnode in r_bucket.sorted_by_rtt():
self._replacement_queue.add(r_rnode)
if r_bucket.there_is_room():
r_bucket.add(rnode)
else:
worst_rnode = self._worst_rnode(r_bucket.rnodes)
if worst_rnode:
# Replace worst node in replacement table
r_bucket.remove(worst_rnode)
r_bucket.add(rnode)
# Node is not in main table
rnode = r_bucket.get_rnode(node_)
if rnode:
# Node in replacement table: just update rnode
self._update_rnode_on_timeout(rnode)
return []
def get_closest_rnodes(self, log_distance, num_nodes, exclude_myself):
if not num_nodes:
num_nodes = NODES_PER_BUCKET[log_distance]
return self.table.get_closest_rnodes(log_distance, num_nodes,
exclude_myself)
def get_main_rnodes(self):
return self.table.get_main_rnodes()
def print_stats(self):
self.table.print_stats()
def _update_rnode_on_query_received(self, rnode):
"""Register a query from node.
You should call this method when receiving a query from this node.
"""
current_time = time.time()
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout += 1
rnode.num_queries += 1
rnode.add_event(current_time, node.QUERY)
rnode.last_seen = current_time
def _update_rnode_on_response_received(self, rnode, rtt):
"""Register a reply from rnode.
You should call this method when receiving a response from this rnode.
"""
rnode.rtt = rtt
current_time = time.time()
#rnode._reset_refresh_task()
if rnode.in_quarantine:
rnode.in_quarantine = \
rnode.last_action_ts < current_time - QUARANTINE_PERIOD
rnode.last_action_ts = current_time
rnode.num_responses += 1
rnode.add_event(time.time(), node.RESPONSE)
rnode.last_seen = current_time
def _update_rnode_on_timeout(self, rnode):
"""Register a timeout for this rnode.
You should call this method when getting a timeout for this node.
"""
rnode.last_action_ts = time.time()
rnode.msgs_since_timeout = 0
rnode.num_timeouts += 1
rnode.add_event(time.time(), node.TIMEOUT)
def _worst_rnode(self, rnodes):
max_num_timeouts = -1
worst_rnode_so_far = None
for rnode in rnodes:
num_timeouots = rnode.timeouts_in_a_row()
if num_timeouots >= max_num_timeouts:
max_num_timeouts = num_timeouots
worst_rnode_so_far = rnode
return worst_rnode_so_far
