예제 #1
0
    def test_add_node(self):
        '''It should add a node'''

        node = Node(KeyBytes(), ('10.0.0.0', 8000))
        rt = RoutingTable()

        rt.node_update(node)
        self.assertTrue(node in rt)
예제 #2
0
    def test_full_bucket(self):
        '''It should raise exception when the 8th bucket is full'''

        rt = RoutingTable()

        for i in range(Bucket.MAX_BUCKET_SIZE + 1):
            key = KeyBytes(b'\x00' \
                + os.urandom(KeyBytes.BIT_SIZE // 8 - 1))
            node = Node(key,
                ('10.0.0.0', random.randint(1024, 10000)))

            if i == 20:
                self.assertRaises(BucketFullError, rt.node_update, node)
            else:
                rt.node_update(node)
예제 #3
0
    def __init__(self, event_reactor, kvp_table, node_id=None, network=None,
    download_slot=None):
        '''Init

        :Parameters:
            event_reactor : :class:`.EventReactor`
                The Event Reactor
            kvp_table : :class:`.KVPTable`
                The storage
            node_id : :class:`.KeyBytes`
                A key to be used as the node id.
        '''

        EventReactorMixin.__init__(self, event_reactor)
        self._network = network or Network(event_reactor)
        self._network.receive_callback = self._receive_callback
        self._routing_table = RoutingTable()
        self._key = node_id or KeyBytes()
        self._pool_executor = WrappedThreadPoolExecutor(
            Network.DEFAULT_POOL_SIZE / 2, event_reactor)
        self._kvp_table = kvp_table
        self._event_scheduler = EventScheduler(event_reactor)
        self._refresh_timer_id = EventID(self, 'Refresh')
        self._download_slot = download_slot or FnTaskSlot()

        self._setup_timers()
예제 #4
0
class DHTNetwork(EventReactorMixin):
    '''The distributed hash table network

    :CVariables:
        NETWORK_ID
            The unique network id reserved only use in the Bytestag network.
    '''

    NETWORK_ID = 'BYTESTAG'
    MAX_VALUE_SIZE = 1048576  # 1 MB
    NETWORK_PARALLELISM = 3  # constant alpha
    TIME_EXPIRE = 86490  # seconds. time-to-live from original publication date
    TIME_REFRESH = 3600  # seconds. time to refresh unaccessed bucket
    TIME_REPLICATE = 3600  # seconds. interval between replication events
    TIME_REPUBLISH = 86400  # seconds. time after original publisher must
    # republish

    def __init__(self, event_reactor, kvp_table, node_id=None, network=None,
    download_slot=None):
        '''Init

        :Parameters:
            event_reactor : :class:`.EventReactor`
                The Event Reactor
            kvp_table : :class:`.KVPTable`
                The storage
            node_id : :class:`.KeyBytes`
                A key to be used as the node id.
        '''

        EventReactorMixin.__init__(self, event_reactor)
        self._network = network or Network(event_reactor)
        self._network.receive_callback = self._receive_callback
        self._routing_table = RoutingTable()
        self._key = node_id or KeyBytes()
        self._pool_executor = WrappedThreadPoolExecutor(
            Network.DEFAULT_POOL_SIZE / 2, event_reactor)
        self._kvp_table = kvp_table
        self._event_scheduler = EventScheduler(event_reactor)
        self._refresh_timer_id = EventID(self, 'Refresh')
        self._download_slot = download_slot or FnTaskSlot()

        self._setup_timers()

    def _setup_timers(self):
        self._event_scheduler.add_periodic(DHTNetwork.TIME_REFRESH / 4,
            self._refresh_timer_id)
        self._event_reactor.register_handler(self._refresh_timer_id,
            self._refresh_buckets)

    @property
    def routing_table(self):
        '''The routing table

        :rtype: :class:`.RoutingTable`
        '''

        return self._routing_table

    @property
    def key(self):
        '''The node id

        :rtype: :class:`.KeyBytes`
        '''

        return self._key

    @property
    def node(self):
        '''The node info

        :rtype: `Node`
        '''

        return Node(self._key, self.address)

    @property
    def address(self):
        '''The address of the server

        :return: A ``tuple`` holding host and port number.
        :rtype: ``tuple``
        '''

        return self._network.server_address

    @property
    def download_slot(self):
        '''The :class:`.FnTaskSlot` which holds
        :class:`.ReadStoreFromNodeTask`.'''

        return self._download_slot

    def _template_dict(self):
        '''Return a new dict holding common stuff like network id'''

        d = {
            JSONKeys.NETWORK_ID: DHTNetwork.NETWORK_ID,
            JSONKeys.NODE_ID: self._key.base64,
        }

        return d

    def _receive_callback(self, data_packet):
        '''An incoming packet callback'''

        dict_obj = data_packet.dict_obj

        if dict_obj.get(JSONKeys.NETWORK_ID) != DHTNetwork.NETWORK_ID:
            _logger.debug('Unknown network id, discarding. %s←%s',
                self.address, data_packet.address)
            return

        self._update_routing_table_from_data_packet(data_packet)

        rpc_name = dict_obj.get(JSONKeys.RPC)

        rpc_map = {
            JSONKeys.RPCs.PING: self._received_ping_rpc,
            JSONKeys.RPCs.FIND_NODE: self._received_find_node_rpc,
            JSONKeys.RPCs.FIND_VALUE: self._received_find_value_rpc,
            JSONKeys.RPCs.GET_VALUE: self._received_get_value_rpc,
            JSONKeys.RPCs.STORE: self._received_store_rpc,
        }

        fn = rpc_map.get(rpc_name)

        if fn:
            _logger.debug('Got rpc %s', rpc_name)
            fn(data_packet)
        else:
            _logger.debug('Received unknown rpc %s', rpc_name)

    def join_network(self, address):
        '''Join the network

        :rtype: :class:`JoinNetworkTask`
        :return: A future that returns ``bool``. If ``True``, the join was
            successful.
        '''

        _logger.debug('Join %s→%s', self.address, address)

        join_network_task = JoinNetworkTask(self, address)

        self._pool_executor.submit(join_network_task)

        return join_network_task

    def ping_address(self, address):
        '''Ping an address

        :rtype: :class:`PingTask`
        :return: A future which returns ``bool`` or a tuple of
            (``float``, `Node`). If a tuple is returned,
            the ping was successful. The items represents the ping time and
            the node.
        '''

        _logger.debug('Ping %s→%s', self.address, address)

        ping_task = PingTask(address, self)

        self._pool_executor.submit(ping_task)

        return ping_task

    def ping_node(self, node):
        '''Ping a node

        :see: `ping_address`
        :rtype: :class:`PingTask`
        '''

        return self.ping_address(node.address)

    def _received_ping_rpc(self, data_packet):
        '''Ping RPC callback'''

        _logger.debug('Pong %s→%s', self.address, data_packet.address)

        d = self._template_dict()

        self._network.send_answer_reply(data_packet, d)

    def find_nodes_from_node(self, node, key):
        '''Find the closest nodes to a key

        :rtype: :class:`FindNodesFromNodeTask`
        :return: A future which returns a `NodeList` or ``None``.
        '''

        _logger.debug('Find node %s→%s %s', self.node, node, key)

        find_nodes_from_node_task = FindNodesFromNodeTask(self, node, key)

        self._pool_executor.submit(find_nodes_from_node_task)

        return find_nodes_from_node_task

    def find_value_from_node(self, node, key, index=None):
        '''Ask a node about values for a key

        :Parameters:
            node: `Node`
                The node to be contacted
            key: :class:`.KeyBytes`
                The key of the value
            index: :class:`.KeyBytes`, ``None``
                If given, the request will be filtered to that given index.

        :rtype: :class:`FindValueFromNodeTask`
        :return: A future which returns a `FindValueFromNodeResult` or
            ``None``.
        '''

        _logger.debug('Find value %s:%s %s→%s', key, index, self.node,
            node)

        find_value_from_node_task = FindValueFromNodeTask(self,
            node, key, index)

        self._pool_executor.submit(find_value_from_node_task)

        return find_value_from_node_task

    def _received_find_node_rpc(self, data_packet):
        '''Find node RPC callback'''

        _logger.debug('Find node %s←%s', self.address,
            data_packet.address)

        key_obj = KeyBytes.new_silent(data_packet.dict_obj.get(JSONKeys.KEY))

        if not key_obj:
            _logger.debug('Find node %s←%s bad key', self.address,
            data_packet.address)
            return

        self._reply_find_node(data_packet, key_obj)

    def _reply_find_node(self, data_packet, key_obj):
        '''Reply to a find node rpc'''

        nodes = self._routing_table.get_close_nodes(key_obj,
            Bucket.MAX_BUCKET_SIZE)
        node_list = NodeList(nodes).to_json_dumpable()
        d = self._template_dict()
        d[JSONKeys.NODES] = node_list

        _logger.debug('Find node reply %s→%s len=%d',
            self.address, data_packet.address, len(node_list))
        self._network.send_answer_reply(data_packet, d)

    def _received_find_value_rpc(self, data_packet):
        '''Find value rpc callback'''

        _logger.debug('Find value %s←%s', self.address,
            data_packet.address)

        key = KeyBytes.new_silent(data_packet.dict_obj.get(JSONKeys.KEY,
            'fake'))
        index = KeyBytes.new_silent(data_packet.dict_obj.get(JSONKeys.INDEX))

        if not key:
            _logger.debug('Find value %s←%s bad key', self.address,
            data_packet.address)
            return

        _logger.debug('Find value %s←%s k=%s i=%s', self.address,
            data_packet.address, key, index)

        kvpid = KVPID(key, index)

        if index and kvpid in self._kvp_table:
            kvp_record = self._kvp_table.record(kvpid)

            d = self._template_dict()
            d[JSONKeys.VALUES] = KVPExchangeInfoList([
                KVPExchangeInfo.from_kvp_record(kvp_record)
            ]).to_json_dumpable()

            self._network.send_answer_reply(data_packet, d)
        elif self._kvp_table.indices(key):
            kvp_record_list = self._kvp_table.records_by_key(key)

            d = self._template_dict()
            d[JSONKeys.VALUES] = KVPExchangeInfoList.from_kvp_record_list(
                kvp_record_list).to_json_dumpable()

            self._network.send_answer_reply(data_packet, d)
        else:
            self._reply_find_node(data_packet, key)

    def find_node_shortlist(self, key):
        '''Return nodes close to a key

        :rtype: :class:`FindShortlistTask`
        '''

        _logger.debug('Find nodes k=%s', key)

        find_shortlist_task = FindShortlistTask(self, key,
            find_nodes=True)

        self._pool_executor.submit(find_shortlist_task)

        return find_shortlist_task

    def find_value_shortlist(self, key, index=None):
        '''Return nodes close to a key and may have the value

        :rtype: :class:`FindShortlistTask`
        '''

        _logger.debug('Find value k=%s', key)

        find_shortlist_task = FindShortlistTask(self, key, index=index,
            find_nodes=False)

        self._pool_executor.submit(find_shortlist_task)

        return find_shortlist_task

    def _data_packet_to_node(self, data_packet):
        '''Extract node info from a packet

        :rtype: :class:`Node`
        '''

        address = data_packet.address

        try:
            node_key = KeyBytes(data_packet.dict_obj.get(JSONKeys.NODE_ID))
        except Exception as e:
            _logger.debug('Ignore key error %s', e)
            return

        return Node(node_key, address)

    def _update_routing_table_from_data_packet(self, data_packet):
        '''Extract node and update routing table from a data packet'''

        node = self._data_packet_to_node(data_packet)

        if node:
            self._update_routing_table(node)

    def _update_routing_table(self, node):
        '''Update the routing table with this node.

        The node must have contacted us or it has responded.
        '''

        if node.key == self._key:
            _logger.debug('Ignore node %s with our id on routing table update',
                node)
            return

        try:
            self._routing_table.node_update(node)
        except BucketFullError as e:
            bucket = e.bucket
            old_node = e.node

            self._update_full_bucket(bucket, old_node, node)

    @asynchronous(name='update_full_bucket')
    def _update_full_bucket(self, bucket, old_node, new_node):
        '''A full bucket callback that will ping and update the buckets'''

        _logger.debug('Update routing table, bucket=%s full', bucket)

        future = self.ping_node(old_node)
        has_responded = future.result()

        if not has_responded:
            _logger.debug('Bucket %s drop %s add %s', bucket, old_node,
                new_node)
            bucket.keep_new_node()
        else:
            _logger.debug('Bucket %s keep %s ignore %s', bucket, old_node,
                new_node)
            bucket.keep_old_node()

    def get_value_from_node(self, node, key, index=None, offset=None):
        '''Download, from a node, data value associated to the key

        :rtype: :class:`.DownloadTask`
        '''

        transfer_id = self._network.new_sequence_id()
        d = self._template_dict()
        d[JSONKeys.RPC] = JSONKeys.RPCs.GET_VALUE
        d[JSONKeys.KEY] = key.base64
        d[JSONKeys.INDEX] = index.base64 if index else key.base64
        d[JSONKeys.TRANSFER_ID] = transfer_id

        if offset:
            d[JSONKeys.VALUE_OFFSET] = offset

        task = self._network.expect_incoming_transfer(transfer_id)

        _logger.debug('Get value %s→%s transfer_id=%s', self.node, node,
            transfer_id)

        self._network.send(node.address, d)

        return task

    @asynchronous(name='received_get_value_rpc')
    def _received_get_value_rpc(self, data_packet):
        '''Get value rpc calllback'''

        _logger.debug('Get value %s←%s', self.address,
            data_packet.address)

        self._update_routing_table_from_data_packet(data_packet)

        key = KeyBytes.new_silent(data_packet.dict_obj[JSONKeys.KEY])
        index = KeyBytes.new_silent(data_packet.dict_obj[JSONKeys.INDEX])
        transfer_id = data_packet.dict_obj.get(JSONKeys.TRANSFER_ID)

        if not transfer_id:
            _logger.debug('Missing transfer id')
            return

        try:
            offset = data_packet.dict_obj.get(JSONKeys.VALUE_OFFSET, 0)
        except TypeError as e:
            _logger.debug('Offset parse error %s', e)

            return

        kvpid = KVPID(key, index)

        if not kvpid in self._kvp_table:
            _logger.debug('KeyBytes not in cache')
            return

        data = self._kvp_table[kvpid]

        task = self._network.send_bytes(data_packet.address,
            transfer_id, data[offset:])
        bytes_sent = task.result()

        _logger.debug('Sent %d bytes', bytes_sent)

    def store_to_node(self, node, key, index, bytes_, timestamp):
        '''Send data to node.

        :rtype: :class:`StoreToNodeTask`
        '''

        _logger.debug('Store value %s→%s', self.node, node)

        store_to_node_task = StoreToNodeTask(self, node, key, index,
            bytes_, timestamp)

        self._pool_executor.submit(store_to_node_task)

        return store_to_node_task

    @asynchronous(name='received_store_rpc')
    def _received_store_rpc(self, data_packet):
        '''Received store RPC'''

        _logger.debug('Store value %s←%s', self.address, data_packet.address)

        dict_obj = data_packet.dict_obj

        # FIXME: validation
        key = KeyBytes(dict_obj[JSONKeys.KEY])
        index = KeyBytes(dict_obj[JSONKeys.INDEX])
        size = int(dict_obj[JSONKeys.SIZE])
        timestamp = int(dict_obj[JSONKeys.TIMESTAMP])

        d = self._template_dict()
        kvpid = KVPID(key, index)

        if self._kvp_table.is_acceptable(kvpid, size, timestamp):
            transfer_id = self._network.new_sequence_id()

            download_task = self._download_slot.add(
                self._network.expect_incoming_transfer, transfer_id,
                max_size=DHTNetwork.MAX_VALUE_SIZE,
                download_task_class=ReadStoreFromNodeTask)

            download_task.key = kvpid.key
            download_task.index = kvpid.index
            download_task.total_size = size
            d[JSONKeys.TRANSFER_ID] = transfer_id

            self._network.send_answer_reply(data_packet, d)

            _logger.debug('Store value %s←%s begin read', self.address,
                data_packet.address)

            file = download_task.result()

            _logger.debug('Store value %s←%s received data', self.address,
                data_packet.address)

            data = file.read()

            if index.validate_value(data):
                self._kvp_table[kvpid] = data
                kvp_record = self._kvp_table.record(kvpid)
                kvp_record.timestamp = timestamp
                kvp_record.last_update = time.time()
                kvp_record.time_to_live = self._calculate_expiration_time(key)
        else:
            self._network.send_answer_reply(data_packet, d)

    def _calculate_expiration_time(self, key):
        '''Return the expiration time for a given key'''

        bucket_number = compute_bucket_number(self.key, key)
        num_contacts = sum(
            [len(self.routing_table[i]) for i in range(bucket_number)])
        num_bucket_contacts = self._routing_table.count_close(key)

        c = num_contacts + num_bucket_contacts

        if c < Bucket.MAX_BUCKET_SIZE == 0:
            return DHTNetwork.TIME_EXPIRE
        else:
            return DHTNetwork.TIME_EXPIRE / math.exp(
                c / Bucket.MAX_BUCKET_SIZE)

    @asynchronous(name='refresh buckets')
    def _refresh_buckets(self, event_id):
        for bucket in self._routing_table.buckets:
            if bucket.last_update + DHTNetwork.TIME_REFRESH < time.time():
                key = random_bucket_key(self.node.key, bucket.number)
                task = self.find_node_shortlist(key)
                task.result()

    def store_value(self, key, index):
        '''Publish or replicate value to nodes.

        :rtype: :class:`StoreValueTask`
        '''

        _logger.debug('Store value %s:%s', key, index)

        store_value_task = StoreValueTask(self, key, index)

        self._pool_executor.submit(store_value_task)

        return store_value_task

    def get_value(self, key, index):
        get_value_task = GetValueTask(self, key, index)

        def f():
            self._pool_executor.submit(get_value_task)

            return get_value_task

        self._download_slot.add(f)

        return get_value_task