def test_add_node(self):
'''It should add a node'''
node = Node(KeyBytes(), ('10.0.0.0', 8000))
rt = RoutingTable()
rt.node_update(node)
self.assertTrue(node in rt)
def test_full_bucket(self):
'''It should raise exception when the 8th bucket is full'''
rt = RoutingTable()
for i in range(Bucket.MAX_BUCKET_SIZE + 1):
key = KeyBytes(b'\x00' \
+ os.urandom(KeyBytes.BIT_SIZE // 8 - 1))
node = Node(key,
('10.0.0.0', random.randint(1024, 10000)))
if i == 20:
self.assertRaises(BucketFullError, rt.node_update, node)
else:
rt.node_update(node)
def __init__(self, event_reactor, kvp_table, node_id=None, network=None,
download_slot=None):
'''Init
:Parameters:
event_reactor : :class:`.EventReactor`
The Event Reactor
kvp_table : :class:`.KVPTable`
The storage
node_id : :class:`.KeyBytes`
A key to be used as the node id.
'''
EventReactorMixin.__init__(self, event_reactor)
self._network = network or Network(event_reactor)
self._network.receive_callback = self._receive_callback
self._routing_table = RoutingTable()
self._key = node_id or KeyBytes()
self._pool_executor = WrappedThreadPoolExecutor(
Network.DEFAULT_POOL_SIZE / 2, event_reactor)
self._kvp_table = kvp_table
self._event_scheduler = EventScheduler(event_reactor)
self._refresh_timer_id = EventID(self, 'Refresh')
self._download_slot = download_slot or FnTaskSlot()
self._setup_timers()
class DHTNetwork(EventReactorMixin):
'''The distributed hash table network
:CVariables:
NETWORK_ID
The unique network id reserved only use in the Bytestag network.
'''
NETWORK_ID = 'BYTESTAG'
MAX_VALUE_SIZE = 1048576 # 1 MB
NETWORK_PARALLELISM = 3 # constant alpha
TIME_EXPIRE = 86490 # seconds. time-to-live from original publication date
TIME_REFRESH = 3600 # seconds. time to refresh unaccessed bucket
TIME_REPLICATE = 3600 # seconds. interval between replication events
TIME_REPUBLISH = 86400 # seconds. time after original publisher must
# republish
def __init__(self, event_reactor, kvp_table, node_id=None, network=None,
download_slot=None):
'''Init
:Parameters:
event_reactor : :class:`.EventReactor`
The Event Reactor
kvp_table : :class:`.KVPTable`
The storage
node_id : :class:`.KeyBytes`
A key to be used as the node id.
'''
EventReactorMixin.__init__(self, event_reactor)
self._network = network or Network(event_reactor)
self._network.receive_callback = self._receive_callback
self._routing_table = RoutingTable()
self._key = node_id or KeyBytes()
self._pool_executor = WrappedThreadPoolExecutor(
Network.DEFAULT_POOL_SIZE / 2, event_reactor)
self._kvp_table = kvp_table
self._event_scheduler = EventScheduler(event_reactor)
self._refresh_timer_id = EventID(self, 'Refresh')
self._download_slot = download_slot or FnTaskSlot()
self._setup_timers()
def _setup_timers(self):
self._event_scheduler.add_periodic(DHTNetwork.TIME_REFRESH / 4,
self._refresh_timer_id)
self._event_reactor.register_handler(self._refresh_timer_id,
self._refresh_buckets)
@property
def routing_table(self):
'''The routing table
:rtype: :class:`.RoutingTable`
'''
return self._routing_table
@property
def key(self):
'''The node id
:rtype: :class:`.KeyBytes`
'''
return self._key
@property
def node(self):
'''The node info
:rtype: `Node`
'''
return Node(self._key, self.address)
@property
def address(self):
'''The address of the server
:return: A ``tuple`` holding host and port number.
:rtype: ``tuple``
'''
return self._network.server_address
@property
def download_slot(self):
'''The :class:`.FnTaskSlot` which holds
:class:`.ReadStoreFromNodeTask`.'''
return self._download_slot
def _template_dict(self):
'''Return a new dict holding common stuff like network id'''
d = {
JSONKeys.NETWORK_ID: DHTNetwork.NETWORK_ID,
JSONKeys.NODE_ID: self._key.base64,
}
return d
def _receive_callback(self, data_packet):
'''An incoming packet callback'''
dict_obj = data_packet.dict_obj
if dict_obj.get(JSONKeys.NETWORK_ID) != DHTNetwork.NETWORK_ID:
_logger.debug('Unknown network id, discarding. %s←%s',
self.address, data_packet.address)
return
self._update_routing_table_from_data_packet(data_packet)
rpc_name = dict_obj.get(JSONKeys.RPC)
rpc_map = {
JSONKeys.RPCs.PING: self._received_ping_rpc,
JSONKeys.RPCs.FIND_NODE: self._received_find_node_rpc,
JSONKeys.RPCs.FIND_VALUE: self._received_find_value_rpc,
JSONKeys.RPCs.GET_VALUE: self._received_get_value_rpc,
JSONKeys.RPCs.STORE: self._received_store_rpc,
}
fn = rpc_map.get(rpc_name)
if fn:
_logger.debug('Got rpc %s', rpc_name)
fn(data_packet)
else:
_logger.debug('Received unknown rpc %s', rpc_name)
def join_network(self, address):
'''Join the network
:rtype: :class:`JoinNetworkTask`
:return: A future that returns ``bool``. If ``True``, the join was
successful.
'''
_logger.debug('Join %s→%s', self.address, address)
join_network_task = JoinNetworkTask(self, address)
self._pool_executor.submit(join_network_task)
return join_network_task
def ping_address(self, address):
'''Ping an address
:rtype: :class:`PingTask`
:return: A future which returns ``bool`` or a tuple of
(``float``, `Node`). If a tuple is returned,
the ping was successful. The items represents the ping time and
the node.
'''
_logger.debug('Ping %s→%s', self.address, address)
ping_task = PingTask(address, self)
self._pool_executor.submit(ping_task)
return ping_task
def ping_node(self, node):
'''Ping a node
:see: `ping_address`
:rtype: :class:`PingTask`
'''
return self.ping_address(node.address)
def _received_ping_rpc(self, data_packet):
'''Ping RPC callback'''
_logger.debug('Pong %s→%s', self.address, data_packet.address)
d = self._template_dict()
self._network.send_answer_reply(data_packet, d)
def find_nodes_from_node(self, node, key):
'''Find the closest nodes to a key
:rtype: :class:`FindNodesFromNodeTask`
:return: A future which returns a `NodeList` or ``None``.
'''
_logger.debug('Find node %s→%s %s', self.node, node, key)
find_nodes_from_node_task = FindNodesFromNodeTask(self, node, key)
self._pool_executor.submit(find_nodes_from_node_task)
return find_nodes_from_node_task
def find_value_from_node(self, node, key, index=None):
'''Ask a node about values for a key
:Parameters:
node: `Node`
The node to be contacted
key: :class:`.KeyBytes`
The key of the value
index: :class:`.KeyBytes`, ``None``
If given, the request will be filtered to that given index.
:rtype: :class:`FindValueFromNodeTask`
:return: A future which returns a `FindValueFromNodeResult` or
``None``.
'''
_logger.debug('Find value %s:%s %s→%s', key, index, self.node,
node)
find_value_from_node_task = FindValueFromNodeTask(self,
node, key, index)
self._pool_executor.submit(find_value_from_node_task)
return find_value_from_node_task
def _received_find_node_rpc(self, data_packet):
'''Find node RPC callback'''
_logger.debug('Find node %s←%s', self.address,
data_packet.address)
key_obj = KeyBytes.new_silent(data_packet.dict_obj.get(JSONKeys.KEY))
if not key_obj:
_logger.debug('Find node %s←%s bad key', self.address,
data_packet.address)
return
self._reply_find_node(data_packet, key_obj)
def _reply_find_node(self, data_packet, key_obj):
'''Reply to a find node rpc'''
nodes = self._routing_table.get_close_nodes(key_obj,
Bucket.MAX_BUCKET_SIZE)
node_list = NodeList(nodes).to_json_dumpable()
d = self._template_dict()
d[JSONKeys.NODES] = node_list
_logger.debug('Find node reply %s→%s len=%d',
self.address, data_packet.address, len(node_list))
self._network.send_answer_reply(data_packet, d)
def _received_find_value_rpc(self, data_packet):
'''Find value rpc callback'''
_logger.debug('Find value %s←%s', self.address,
data_packet.address)
key = KeyBytes.new_silent(data_packet.dict_obj.get(JSONKeys.KEY,
'fake'))
index = KeyBytes.new_silent(data_packet.dict_obj.get(JSONKeys.INDEX))
if not key:
_logger.debug('Find value %s←%s bad key', self.address,
data_packet.address)
return
_logger.debug('Find value %s←%s k=%s i=%s', self.address,
data_packet.address, key, index)
kvpid = KVPID(key, index)
if index and kvpid in self._kvp_table:
kvp_record = self._kvp_table.record(kvpid)
d = self._template_dict()
d[JSONKeys.VALUES] = KVPExchangeInfoList([
KVPExchangeInfo.from_kvp_record(kvp_record)
]).to_json_dumpable()
self._network.send_answer_reply(data_packet, d)
elif self._kvp_table.indices(key):
kvp_record_list = self._kvp_table.records_by_key(key)
d = self._template_dict()
d[JSONKeys.VALUES] = KVPExchangeInfoList.from_kvp_record_list(
kvp_record_list).to_json_dumpable()
self._network.send_answer_reply(data_packet, d)
else:
self._reply_find_node(data_packet, key)
def find_node_shortlist(self, key):
'''Return nodes close to a key
:rtype: :class:`FindShortlistTask`
'''
_logger.debug('Find nodes k=%s', key)
find_shortlist_task = FindShortlistTask(self, key,
find_nodes=True)
self._pool_executor.submit(find_shortlist_task)
return find_shortlist_task
def find_value_shortlist(self, key, index=None):
'''Return nodes close to a key and may have the value
:rtype: :class:`FindShortlistTask`
'''
_logger.debug('Find value k=%s', key)
find_shortlist_task = FindShortlistTask(self, key, index=index,
find_nodes=False)
self._pool_executor.submit(find_shortlist_task)
return find_shortlist_task
def _data_packet_to_node(self, data_packet):
'''Extract node info from a packet
:rtype: :class:`Node`
'''
address = data_packet.address
try:
node_key = KeyBytes(data_packet.dict_obj.get(JSONKeys.NODE_ID))
except Exception as e:
_logger.debug('Ignore key error %s', e)
return
return Node(node_key, address)
def _update_routing_table_from_data_packet(self, data_packet):
'''Extract node and update routing table from a data packet'''
node = self._data_packet_to_node(data_packet)
if node:
self._update_routing_table(node)
def _update_routing_table(self, node):
'''Update the routing table with this node.
The node must have contacted us or it has responded.
'''
if node.key == self._key:
_logger.debug('Ignore node %s with our id on routing table update',
node)
return
try:
self._routing_table.node_update(node)
except BucketFullError as e:
bucket = e.bucket
old_node = e.node
self._update_full_bucket(bucket, old_node, node)
@asynchronous(name='update_full_bucket')
def _update_full_bucket(self, bucket, old_node, new_node):
'''A full bucket callback that will ping and update the buckets'''
_logger.debug('Update routing table, bucket=%s full', bucket)
future = self.ping_node(old_node)
has_responded = future.result()
if not has_responded:
_logger.debug('Bucket %s drop %s add %s', bucket, old_node,
new_node)
bucket.keep_new_node()
else:
_logger.debug('Bucket %s keep %s ignore %s', bucket, old_node,
new_node)
bucket.keep_old_node()
def get_value_from_node(self, node, key, index=None, offset=None):
'''Download, from a node, data value associated to the key
:rtype: :class:`.DownloadTask`
'''
transfer_id = self._network.new_sequence_id()
d = self._template_dict()
d[JSONKeys.RPC] = JSONKeys.RPCs.GET_VALUE
d[JSONKeys.KEY] = key.base64
d[JSONKeys.INDEX] = index.base64 if index else key.base64
d[JSONKeys.TRANSFER_ID] = transfer_id
if offset:
d[JSONKeys.VALUE_OFFSET] = offset
task = self._network.expect_incoming_transfer(transfer_id)
_logger.debug('Get value %s→%s transfer_id=%s', self.node, node,
transfer_id)
self._network.send(node.address, d)
return task
@asynchronous(name='received_get_value_rpc')
def _received_get_value_rpc(self, data_packet):
'''Get value rpc calllback'''
_logger.debug('Get value %s←%s', self.address,
data_packet.address)
self._update_routing_table_from_data_packet(data_packet)
key = KeyBytes.new_silent(data_packet.dict_obj[JSONKeys.KEY])
index = KeyBytes.new_silent(data_packet.dict_obj[JSONKeys.INDEX])
transfer_id = data_packet.dict_obj.get(JSONKeys.TRANSFER_ID)
if not transfer_id:
_logger.debug('Missing transfer id')
return
try:
offset = data_packet.dict_obj.get(JSONKeys.VALUE_OFFSET, 0)
except TypeError as e:
_logger.debug('Offset parse error %s', e)
return
kvpid = KVPID(key, index)
if not kvpid in self._kvp_table:
_logger.debug('KeyBytes not in cache')
return
data = self._kvp_table[kvpid]
task = self._network.send_bytes(data_packet.address,
transfer_id, data[offset:])
bytes_sent = task.result()
_logger.debug('Sent %d bytes', bytes_sent)
def store_to_node(self, node, key, index, bytes_, timestamp):
'''Send data to node.
:rtype: :class:`StoreToNodeTask`
'''
_logger.debug('Store value %s→%s', self.node, node)
store_to_node_task = StoreToNodeTask(self, node, key, index,
bytes_, timestamp)
self._pool_executor.submit(store_to_node_task)
return store_to_node_task
@asynchronous(name='received_store_rpc')
def _received_store_rpc(self, data_packet):
'''Received store RPC'''
_logger.debug('Store value %s←%s', self.address, data_packet.address)
dict_obj = data_packet.dict_obj
# FIXME: validation
key = KeyBytes(dict_obj[JSONKeys.KEY])
index = KeyBytes(dict_obj[JSONKeys.INDEX])
size = int(dict_obj[JSONKeys.SIZE])
timestamp = int(dict_obj[JSONKeys.TIMESTAMP])
d = self._template_dict()
kvpid = KVPID(key, index)
if self._kvp_table.is_acceptable(kvpid, size, timestamp):
transfer_id = self._network.new_sequence_id()
download_task = self._download_slot.add(
self._network.expect_incoming_transfer, transfer_id,
max_size=DHTNetwork.MAX_VALUE_SIZE,
download_task_class=ReadStoreFromNodeTask)
download_task.key = kvpid.key
download_task.index = kvpid.index
download_task.total_size = size
d[JSONKeys.TRANSFER_ID] = transfer_id
self._network.send_answer_reply(data_packet, d)
_logger.debug('Store value %s←%s begin read', self.address,
data_packet.address)
file = download_task.result()
_logger.debug('Store value %s←%s received data', self.address,
data_packet.address)
data = file.read()
if index.validate_value(data):
self._kvp_table[kvpid] = data
kvp_record = self._kvp_table.record(kvpid)
kvp_record.timestamp = timestamp
kvp_record.last_update = time.time()
kvp_record.time_to_live = self._calculate_expiration_time(key)
else:
self._network.send_answer_reply(data_packet, d)
def _calculate_expiration_time(self, key):
'''Return the expiration time for a given key'''
bucket_number = compute_bucket_number(self.key, key)
num_contacts = sum(
[len(self.routing_table[i]) for i in range(bucket_number)])
num_bucket_contacts = self._routing_table.count_close(key)
c = num_contacts + num_bucket_contacts
if c < Bucket.MAX_BUCKET_SIZE == 0:
return DHTNetwork.TIME_EXPIRE
else:
return DHTNetwork.TIME_EXPIRE / math.exp(
c / Bucket.MAX_BUCKET_SIZE)
@asynchronous(name='refresh buckets')
def _refresh_buckets(self, event_id):
for bucket in self._routing_table.buckets:
if bucket.last_update + DHTNetwork.TIME_REFRESH < time.time():
key = random_bucket_key(self.node.key, bucket.number)
task = self.find_node_shortlist(key)
task.result()
def store_value(self, key, index):
'''Publish or replicate value to nodes.
:rtype: :class:`StoreValueTask`
'''
_logger.debug('Store value %s:%s', key, index)
store_value_task = StoreValueTask(self, key, index)
self._pool_executor.submit(store_value_task)
return store_value_task
def get_value(self, key, index):
get_value_task = GetValueTask(self, key, index)
def f():
self._pool_executor.submit(get_value_task)
return get_value_task
self._download_slot.add(f)
return get_value_task