Ejemplo n.º 1
0
 def newNodeRequest():
     nonlocal nr_nodes
     with nr_nodes_cv:
         nr_nodes += 1
         DhtNetwork.log("Node joining the DHT.", "(now "+str(nr_nodes)+" nodes on the dht)")
         nr_nodes_cv.notify()
     random.choice(wb.procs).sendNodesRequest(DhtNetworkSubProcess.NEW_NODE_REQ)
Ejemplo n.º 2
0
    def _trigger_dp(self, trigger_nodes, _hash, count=1):
        """
        Triggers the data persistence over time. In order to this, `count` nodes
        are created with an id around the hash of a value.

        @param trigger_nodes: List of created nodes. The nodes created in this
                              function are append to this list.
        @type  trigger_nodes: list
        @param _hash: Is the id of the value around which creating nodes.
        @type  _hash: InfoHash
        @param count: The number of nodes to create with id around the id of
                      value.
        @type  count: int
        """
        _hash_str = _hash.toString().decode()
        _hash_int = int(_hash_str, 16)
        for i in range(int(-count/2), int(count/2)+1):
            _hash_str = '{:40x}'.format(_hash_int + i)
            config = DhtConfig()
            config.setNodeId(InfoHash(_hash_str.encode()))
            n = DhtRunner()
            n.run(config=config)
            n.bootstrap(self._bootstrap.ip4,
                        str(self._bootstrap.port))
            DhtNetwork.log('Node','['+_hash_str+']',
                           'started around', _hash.toString().decode()
                           if n.isRunning() else
                           'failed to start...'
            )
            trigger_nodes.append(n)
Ejemplo n.º 3
0
 def shutdownNodeRequest():
     nonlocal nr_nodes
     with nr_nodes_cv:
         nr_nodes -= 1
         DhtNetwork.log("Node quitting the DHT.", "(now "+str(nr_nodes)+" nodes on the dht)")
         nr_nodes_cv.notify()
     random.choice(wb.procs).sendNodesRequest(DhtNetworkSubProcess.SHUTDOWN_NODE_REQ)
Ejemplo n.º 4
0
 def putRequest():
     nonlocal hashes, values, nr_values
     nr_values += 1
     DhtNetwork.log("Random value put on the DHT.",
                    "(now " + str(nr_values) + " values on the dht)")
     random.choice(wb.procs).sendNodePutRequest(
         random.choice(hashes).toString(), random.choice(values))
Ejemplo n.º 5
0
    def _trigger_dp(self, trigger_nodes, _hash, count=1):
        """
        Triggers the data persistence over time. In order to this, `count` nodes
        are created with an id around the hash of a value.

        @param trigger_nodes: List of created nodes. The nodes created in this
                              function are append to this list.
        @type  trigger_nodes: list
        @param _hash: Is the id of the value around which creating nodes.
        @type  _hash: InfoHash
        @param count: The number of nodes to create with id around the id of
                      value.
        @type  count: int
        """
        _hash_str = _hash.toString().decode()
        _hash_int = int(_hash_str, 16)
        for i in range(int(-count / 2), int(count / 2) + 1):
            _hash_str = '{:40x}'.format(_hash_int + i)
            config = DhtConfig()
            config.setNodeId(InfoHash(_hash_str.encode()))
            n = DhtRunner()
            n.run(config=config)
            n.bootstrap(self._bootstrap.ip4, str(self._bootstrap.port))
            DhtNetwork.log(
                'Node', '[' + _hash_str + ']', 'started around',
                _hash.toString().decode()
                if n.isRunning() else 'failed to start...')
            trigger_nodes.append(n)
Ejemplo n.º 6
0
 def _dhtPut(self, producer, _hash, *values):
     for val in values:
         with FeatureTest.lock:
             DhtNetwork.log('[PUT]: %s' % val)
             FeatureTest.done += 1
             producer.put(_hash, val, DhtFeatureTest.putDoneCb)
             while FeatureTest.done > 0:
                 FeatureTest.lock.wait()
Ejemplo n.º 7
0
 def shutdownNodeRequest():
     nonlocal nr_nodes
     with nr_nodes_cv:
         nr_nodes -= 1
         DhtNetwork.log("Node quitting the DHT.",
                        "(now " + str(nr_nodes) + " nodes on the dht)")
         nr_nodes_cv.notify()
     random.choice(wb.procs).sendNodesRequest(
         DhtNetworkSubProcess.SHUTDOWN_NODE_REQ)
Ejemplo n.º 8
0
 def newNodeRequest():
     nonlocal nr_nodes
     with nr_nodes_cv:
         nr_nodes += 1
         DhtNetwork.log("Node joining the DHT.",
                        "(now " + str(nr_nodes) + " nodes on the dht)")
         nr_nodes_cv.notify()
     random.choice(wb.procs).sendNodesRequest(
         DhtNetworkSubProcess.NEW_NODE_REQ)
Ejemplo n.º 9
0
 def _dhtGet(self, consumer, _hash):
     DhtFeatureTest.foreignValues = []
     DhtFeatureTest.foreignNodes = []
     with FeatureTest.lock:
         FeatureTest.done += 1
         DhtNetwork.log('[GET]:', _hash.toString())
         consumer.get(_hash, DhtFeatureTest.getcb, DhtFeatureTest.getDoneCb)
         while FeatureTest.done > 0:
             FeatureTest.lock.wait()
Ejemplo n.º 10
0
 def _dhtPut(self, producer, _hash, *values):
     for val in values:
         with FeatureTest.lock:
             vstr = val.__str__()[:100]
             DhtNetwork.log('[PUT]: %s' % vstr + ("..." if len(vstr) > 100 else ""))
             FeatureTest.done += 1
             while FeatureTest.done > 0:
                 producer.put(_hash, val, DhtFeatureTest.putDoneCb)
                 FeatureTest.lock.wait()
Ejemplo n.º 11
0
 def _dhtGet(self, consumer, _hash):
     DhtFeatureTest.foreignValues = []
     DhtFeatureTest.foreignNodes = []
     with FeatureTest.lock:
         FeatureTest.done += 1
         while FeatureTest.done > 0:
             DhtNetwork.log('[GET]:', _hash.toString())
             consumer.get(_hash, DhtFeatureTest.getcb, DhtFeatureTest.getDoneCb)
             FeatureTest.lock.wait()
Ejemplo n.º 12
0
 def donecb(ok, nodes):
     nonlocal bootstrap, lock, done, times
     t = time.time()-start
     with lock:
         if not ok:
             DhtNetwork.log("failed !")
         times.append(t)
         done -= 1
         lock.notify()
Ejemplo n.º 13
0
 def donecb(ok, nodes):
     nonlocal bootstrap, lock, done, times
     t = time.time()-start
     with lock:
         if not ok:
             DhtNetwork.log("failed !")
         times.append(t)
         done -= 1
         lock.notify()
Ejemplo n.º 14
0
 def getDoneCb(ok, nodes):
     with FeatureTest.lock:
         if not ok:
             DhtNetwork.log("[GET]: failed!")
         else:
             for node in nodes:
                 if not node.getNode().isExpired():
                     DhtFeatureTest.foreignNodes.append(node.getId().toString())
         FeatureTest.done -= 1
         FeatureTest.lock.notify()
Ejemplo n.º 15
0
 def _dhtPut(self, producer, _hash, *values):
     with FeatureTest.lock:
         for val in values:
             vstr = val.__str__()[:100]
             DhtNetwork.log('[PUT]:', _hash.toString(), '->',
                            vstr + ("..." if len(vstr) > 100 else ""))
             FeatureTest.done += 1
             producer.put(_hash, val, DhtFeatureTest.putDoneCb)
         while FeatureTest.done > 0:
             FeatureTest.lock.wait()
Ejemplo n.º 16
0
 def getDoneCb(ok, nodes):
     with FeatureTest.lock:
         if not ok:
             DhtNetwork.log("[GET]: failed!")
         else:
             for node in nodes:
                 if not node.getNode().isExpired():
                     DhtFeatureTest.foreignNodes.append(node.getId().toString())
         FeatureTest.done -= 1
         FeatureTest.lock.notify()
Ejemplo n.º 17
0
    def _insertTest(self):
        """TODO: Docstring for _massIndexTest.
        """
        bootstrap = self._bootstrap
        bootstrap.resize(2)
        dht = bootstrap.get(1)

        NUM_DIG = max(math.log(self._num_keys, 2) / 4,
                      5)  # at least 5 digit keys.
        keyspec = collections.OrderedDict([('foo', NUM_DIG)])
        pht = Pht(b'foo_index', keyspec, dht)

        DhtNetwork.log('PHT has', pht.MAX_NODE_ENTRY_COUNT,
                       'node' + ('s' if pht.MAX_NODE_ENTRY_COUNT > 1 else ''),
                       'per leaf bucket.')
        keys = [{
            [_ for _ in keyspec.keys()][0]:
            ''.join(random.SystemRandom().choice(string.hexdigits)
                    for _ in range(NUM_DIG)).encode()
        } for n in range(self._num_keys)]
        all_entries = {}

        # Index all entries.
        for key in keys:
            PhtTest.key = key
            with FeatureTest.lock:
                time_taken = timer(pht.insert, key, IndexValue(random_hash()),
                                   PhtTest.insertDoneCb)
                if self._timer:
                    DhtNetwork.log('This insert step took : ', time_taken,
                                   'second')
                FeatureTest.lock.wait()

        time.sleep(1)

        # Recover entries now that the trie is complete.
        for key in keys:
            PhtTest.key = key
            with FeatureTest.lock:
                time_taken = timer(pht.lookup, key, PhtTest.lookupCb,
                                   PhtTest.lookupDoneCb)
                if self._timer:
                    DhtNetwork.log('This lookup step took : ', time_taken,
                                   'second')
                FeatureTest.lock.wait()

            all_entries[PhtTest.prefix] = [
                e.__str__() for e in PhtTest.indexEntries
            ]

        for p in all_entries.keys():
            DhtNetwork.log('All entries under prefix', p, ':')
            DhtNetwork.log(all_entries[p])
        PhtTest.drawTrie(all_entries)
Ejemplo n.º 18
0
    def _insertTest(self):
        """TODO: Docstring for _massIndexTest.
        """
        bootstrap = self._bootstrap
        bootstrap.resize(2)
        dht = bootstrap.get(1)

        NUM_DIG  = max(math.log(self._num_keys, 2)/4, 5) # at least 5 digit keys.
        keyspec = collections.OrderedDict([('foo', NUM_DIG)])
        pht = Pht(b'foo_index', keyspec, dht)

        DhtNetwork.log('PHT has',
                       pht.MAX_NODE_ENTRY_COUNT,
                       'node'+ ('s' if pht.MAX_NODE_ENTRY_COUNT > 1 else ''),
                       'per leaf bucket.')
        keys = [{
            [_ for _ in keyspec.keys()][0] :
            ''.join(random.SystemRandom().choice(string.hexdigits)
                for _ in range(NUM_DIG)).encode()
            } for n in range(self._num_keys)]
        all_entries = {}

        # Index all entries.
        for key in keys:
            PhtTest.key = key
            with FeatureTest.lock:
                time_taken = timer(pht.insert, key, IndexValue(random_hash()), PhtTest.insertDoneCb)
                if self._timer:
                    DhtNetwork.log('This insert step took : ', time_taken, 'second')
                FeatureTest.lock.wait()

        time.sleep(1)

        # Recover entries now that the trie is complete.
        for key in keys:
            PhtTest.key = key
            with FeatureTest.lock:
                time_taken = timer(pht.lookup, key, PhtTest.lookupCb, PhtTest.lookupDoneCb)
                if self._timer:
                    DhtNetwork.log('This lookup step took : ', time_taken, 'second')
                FeatureTest.lock.wait()

            all_entries[PhtTest.prefix] = [e.__str__()
                                           for e in PhtTest.indexEntries]

        for p in all_entries.keys():
            DhtNetwork.log('All entries under prefix', p, ':')
            DhtNetwork.log(all_entries[p])
        PhtTest.drawTrie(all_entries)
Ejemplo n.º 19
0
 def get_bootstrap(self):
     if not self.local_bootstrap:
         self.local_bootstrap = DhtNetwork(
             iface='br' + self.ifname,
             first_bootstrap=False if self.remote_bootstrap else True,
             bootstrap=[(self.remote_bootstrap,
                         self.bs_port)] if self.remote_bootstrap else [])
     return self.local_bootstrap
Ejemplo n.º 20
0
 def lookupCb(vals, prefix):
     PhtTest.indexEntries = list(vals)
     PhtTest.prefix = prefix.decode()
     DhtNetwork.log('Index name: <todo>')
     DhtNetwork.log('Leaf prefix:', prefix)
     for v in vals:
         DhtNetwork.log('[ENTRY]:', v)
Ejemplo n.º 21
0
    def _replaceClusterTest(self):
        """
        It replaces all clusters one after the other.
        """
        clusters = 8

        bootstrap = self.bootstrap

        bootstrap.resize(3)
        consumer = bootstrap.get(1)
        producer = bootstrap.get(2)

        myhash = random_hash()
        local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')]

        self._dhtPut(producer, myhash, *local_values)
        self._dhtGet(consumer, myhash)
        initial_nodes = DhtFeatureTest.foreignNodes

        DhtNetwork.log('Replacing', clusters, 'random clusters successively...')
        for n in range(clusters):
            i = random.randint(0, len(self._workbench.procs)-1)
            proc = self._workbench.procs[i]
            DhtNetwork.log('Replacing', proc)
            proc.sendNodesRequest(DhtNetworkSubProcess.SHUTDOWN_CLUSTER_REQ)
            self._workbench.stop_cluster(i)
            self._workbench.start_cluster(i)

        DhtNetwork.log('[GET]: trying to fetch persistent values')
        self._dhtGet(consumer, myhash)
        new_nodes = set(DhtFeatureTest.foreignNodes) - set(initial_nodes)

        self._result(local_values, new_nodes)
Ejemplo n.º 22
0
    def _replaceClusterTest(self):
        """
        It replaces all clusters one after the other.
        """

        #clusters = opts['clusters'] if 'clusters' in opts else 5
        clusters = 5

        bootstrap = self.bootstrap

        bootstrap.resize(3)
        consumer = bootstrap.get(1)
        producer = bootstrap.get(2)

        myhash = random_hash()
        local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')]

        self._dhtPut(producer, myhash, *local_values)
        self._dhtGet(consumer, myhash)
        initial_nodes = DhtFeatureTest.foreignNodes

        DhtNetwork.log('Replacing', clusters, 'random clusters successively...')
        for n in range(clusters):
            i = random.randint(0, len(self._workbench.procs)-1)
            proc = self._workbench.procs[i]
            DhtNetwork.log('Replacing', proc)
            proc.sendShutdown()
            self._workbench.stop_cluster(i)
            self._workbench.start_cluster(i)

        DhtNetwork.log('[GET]: trying to fetch persistent values')
        self._dhtGet(consumer, myhash)
        new_nodes = set(DhtFeatureTest.foreignNodes) - set(initial_nodes)

        self._result(local_values, new_nodes)
Ejemplo n.º 23
0
 def lookupCb(vals, prefix):
     PhtTest.indexEntries = list(vals)
     PhtTest.prefix = prefix.decode()
     DhtNetwork.log('Index name: <todo>')
     DhtNetwork.log('Leaf prefix:', prefix)
     for v in vals:
         DhtNetwork.log('[ENTRY]:', v)
Ejemplo n.º 24
0
    def _getsTimesTest(self):
        """
        Tests for performance of the DHT doing multiple get() operation.
        """
        bootstrap = self.bootstrap

        plt.ion()

        fig, axes = plt.subplots(2, 1)
        fig.tight_layout()

        lax = axes[0]
        hax = axes[1]

        lines = None#ax.plot([])
        #plt.ylabel('time (s)')
        hax.set_ylim(0, 2)

        # let the network stabilise
        plt.pause(60)

        #start = time.time()
        times = []

        lock = threading.Condition()
        done = 0

        def getcb(v):
            nonlocal bootstrap
            DhtNetwork.log("found", v)
            return True

        def donecb(ok, nodes):
            nonlocal bootstrap, lock, done, times
            t = time.time()-start
            with lock:
                if not ok:
                    DhtNetwork.log("failed !")
                times.append(t)
                done -= 1
                lock.notify()

        def update_plot():
            nonlocal lines
            while lines:
                l = lines.pop()
                l.remove()
                del l
            lines = plt.plot(times, color='blue')
            plt.draw()

        def run_get():
            nonlocal done
            done += 1
            start = time.time()
            bootstrap.front().get(InfoHash.getRandom(), getcb, lambda ok, nodes: donecb(ok, nodes, start))

        plt.pause(5)

        plt.show()
        update_plot()

        times = []
        for n in range(10):
            self._workbench.replace_cluster()
            plt.pause(2)
            DhtNetwork.log("Getting 50 random hashes succesively.")
            for i in range(50):
                with lock:
                    done += 1
                    start = time.time()
                    bootstrap.front().get(InfoHash.getRandom(), getcb, donecb)
                    while done > 0:
                        lock.wait()
                        update_plot()
                update_plot()
            print("Took", np.sum(times), "mean", np.mean(times), "std", np.std(times), "min", np.min(times), "max", np.max(times))

        print('GET calls timings benchmark test : DONE. '  \
                'Close Matplotlib window for terminating the program.')
        plt.ioff()
        plt.show()
Ejemplo n.º 25
0
    def _delete(self):
        """
        Tests for performance of get() and put() operations on the network while
        deleting around the target hash.
        """


        bootstrap = self.bootstrap

        bootstrap.resize(3)
        consumer = bootstrap.get(1)
        producer = bootstrap.get(2)

        myhash = random_hash()
        local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')]

        for _ in range(max(1, int(self._workbench.node_num/32))):
            self._dhtGet(consumer, myhash)
            DhtNetwork.log("Waiting 15 seconds...")
            time.sleep(15)

            self._dhtPut(producer, myhash, *local_values)

            #checking if values were transfered
            self._dhtGet(consumer, myhash)
            DhtNetwork.log('Values are found on :')
            for node in DhtFeatureTest.foreignNodes:
                DhtNetwork.log(node)

            if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
                if DhtFeatureTest.foreignValues:
                    DhtNetwork.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ',
                            len(local_values), ' values successfully put.')
                else:
                    DhtNetwork.log('[GET]: 0 values successfully put')

            DhtNetwork.log('Removing all nodes hosting target values...')
            for proc in self._workbench.procs:
                DhtNetwork.log('[REMOVE]: sending shutdown request to', proc)
                proc.sendNodesRequest(
                        DhtNetworkSubProcess.SHUTDOWN_NODE_REQ,
                        DhtFeatureTest.foreignNodes
                )
Ejemplo n.º 26
0
    def _multTimeTest(self):
        """
        Multiple put() calls are made from multiple nodes to multiple hashes
        after what a set of 8 nodes is created around each hashes in order to
        enable storage maintenance each nodes. Therefor, this tests will wait 10
        minutes for the nodes to trigger storage maintenance.
        """
        bootstrap = self.bootstrap

        N_PRODUCERS = 16

        hashes = []
        values = [Value(b'foo')]
        nodes = set([])

        # prevents garbage collecting of unused flood nodes during the test.
        flood_nodes = []

        def gottaGetThemAllPokeNodes(nodes=None):
            nonlocal consumer, hashes
            for h in hashes:
                self._dhtGet(consumer, h)
                if nodes is not None:
                    for n in DhtFeatureTest.foreignNodes:
                        nodes.add(n)

        def createNodesAroundHash(_hash, radius=4):
            nonlocal flood_nodes

            _hash_str = _hash.toString().decode()
            _hash_int = int(_hash_str, 16)
            for i in range(-radius, radius+1):
                _hash_str = '{:40x}'.format(_hash_int + i)
                config = DhtConfig()
                config.setNodeId(InfoHash(_hash_str.encode()))
                n = DhtRunner()
                n.run(config=config)
                n.bootstrap(self.bootstrap.ip4,
                            str(self.bootstrap.port))
                flood_nodes.append(n)

        bootstrap.resize(N_PRODUCERS+2)
        consumer = bootstrap.get(1)
        producers = (bootstrap.get(n) for n in range(2,N_PRODUCERS+2))
        for p in producers:
            hashes.append(random_hash())
            self._dhtPut(p, hashes[-1], *values)

        gottaGetThemAllPokeNodes(nodes=nodes)

        DhtNetwork.log("Values are found on:")
        for n in nodes:
            DhtNetwork.log(n)

        DhtNetwork.log("Creating 8 nodes around all of these nodes...")
        for _hash in hashes:
            createNodesAroundHash(_hash)

        DhtNetwork.log('Waiting 10 minutes for normal storage maintenance.')
        time.sleep(10*60)

        DhtNetwork.log('Deleting old nodes from previous search.')
        for proc in self._workbench.procs:
            DhtNetwork.log('[REMOVE]: sending shutdown request to', proc)
            proc.sendNodesRequest(
                DhtNetworkSubProcess.REMOVE_NODE_REQ,
                nodes
            )

        # new consumer (fresh cache)
        bootstrap.resize(N_PRODUCERS+3)
        consumer = bootstrap.get(N_PRODUCERS+2)

        nodes_after_time = set([])
        gottaGetThemAllPokeNodes(nodes=nodes_after_time)
        self._result(values, nodes_after_time - nodes)
Ejemplo n.º 27
0
 def getcb(v):
     nonlocal bootstrap
     DhtNetwork.log("found", v)
     return True
Ejemplo n.º 28
0
 def getcb(value):
     vstr = value.__str__()[:100]
     DhtNetwork.log('[GET]: %s' % vstr + ("..." if len(vstr) > 100 else ""))
     DhtFeatureTest.foreignValues.append(value)
     return True
Ejemplo n.º 29
0
 def insertDoneCb(ok):
     DhtNetwork.log('[INSERT]:', PhtTest.key, "--",
                    "success!" if ok else "Fail...")
     with FeatureTest.lock:
         FeatureTest.lock.notify()
Ejemplo n.º 30
0
 def getcb(v):
     nonlocal bootstrap
     DhtNetwork.log("found", v)
     return True
Ejemplo n.º 31
0
 def lookupDoneCb(ok):
     DhtNetwork.log('[LOOKUP]:', PhtTest.key, "--", "success!" if ok else "Fail...")
     with FeatureTest.lock:
         FeatureTest.lock.notify()
Ejemplo n.º 32
0
 def putRequest():
     nonlocal hashes, values, nr_values
     nr_values += 1
     DhtNetwork.log("Random value put on the DHT.", "(now "+ str(nr_values)+" values on the dht)")
     random.choice(wb.procs).sendNodePutRequest(random.choice(hashes).toString(), random.choice(values))
Ejemplo n.º 33
0
 def _result(self, local_values, new_nodes):
     bootstrap = self.bootstrap
     if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
         DhtNetwork.log('[GET]: Only %s on %s values persisted.' %
                 (len(DhtFeatureTest.foreignValues), len(local_values)))
     else:
         DhtNetwork.log('[GET]: All values successfully persisted.')
     if DhtFeatureTest.foreignValues:
         if new_nodes:
             DhtNetwork.log('Values are newly found on:')
             for node in new_nodes:
                 DhtNetwork.log(node)
             if self._dump_storage:
                 DhtNetwork.log('Dumping all storage log from '\
                               'hosting nodes.')
                 for proc in self._workbench.procs:
                     proc.sendNodesRequest(DhtNetworkSubProcess.DUMP_STORAGE_REQ, DhtFeatureTest.foreignNodes)
         else:
             DhtNetwork.log("Values didn't reach new hosting nodes after shutdown.")
Ejemplo n.º 34
0
 def getcb(value):
     vstr = value.__str__()[:100]
     DhtNetwork.log('[GET]: %s' % vstr + ("..." if len(vstr) > 100 else ""))
     DhtFeatureTest.foreignValues.append(value)
     return True
Ejemplo n.º 35
0
    def _delete(self):
        """
        Tests for performance of get() and put() operations on the network while
        deleting around the target hash.
        """


        bootstrap = self.bootstrap

        bootstrap.resize(3)
        consumer = bootstrap.get(1)
        producer = bootstrap.get(2)

        myhash = random_hash()
        local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')]

        for _ in range(max(1, int(self._workbench.node_num/32))):
            self._dhtGet(consumer, myhash)
            DhtNetwork.log("Waiting 15 seconds...")
            time.sleep(15)

            self._dhtPut(producer, myhash, *local_values)

            #checking if values were transfered
            self._dhtGet(consumer, myhash)
            DhtNetwork.log('Values are found on :')
            for node in DhtFeatureTest.foreignNodes:
                DhtNetwork.log(node)

            if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
                if DhtFeatureTest.foreignValues:
                    DhtNetwork.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ',
                            len(local_values), ' values successfully put.')
                else:
                    DhtNetwork.log('[GET]: 0 values successfully put')

            DhtNetwork.log('Removing all nodes hosting target values...')
            for proc in self._workbench.procs:
                DhtNetwork.log('[REMOVE]: sending shutdown request to', proc)
                proc.sendNodesRequest(
                        DhtNetworkSubProcess.SHUTDOWN_NODE_REQ,
                        DhtFeatureTest.foreignNodes
                )
Ejemplo n.º 36
0
 def putDoneCb(ok, nodes):
     if not ok:
         DhtNetwork.log("[PUT]: failed!")
     with FeatureTest.lock:
         FeatureTest.done -= 1
         FeatureTest.lock.notify()
Ejemplo n.º 37
0
 def insertDoneCb(ok):
     DhtNetwork.log('[INSERT]:', PhtTest.key, "--", "success!" if ok else "Fail...")
     with FeatureTest.lock:
         FeatureTest.lock.notify()
Ejemplo n.º 38
0
    def _deleteTest(self):
        """
        It uses Dht shutdown call from the API to gracefuly finish the nodes one
        after the other.
        """
        bootstrap = self.bootstrap

        ops_count = []

        bootstrap.resize(3)
        consumer = bootstrap.get(1)
        producer = bootstrap.get(2)

        myhash = random_hash()
        local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')]

        self._dhtPut(producer, myhash, *local_values)

        #checking if values were transfered
        self._dhtGet(consumer, myhash)
        if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
            if DhtFeatureTest.foreignValues:
                DhtNetwork.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ',
                        len(local_values), ' values successfully put.')
            else:
                DhtNetwork.log('[GET]: 0 values successfully put')


        if DhtFeatureTest.foreignValues and DhtFeatureTest.foreignNodes:
            DhtNetwork.log('Values are found on :')
            for node in DhtFeatureTest.foreignNodes:
                DhtNetwork.log(node)

            #DhtNetwork.log("Waiting a minute for the network to settle down.")
            #time.sleep(60)

            for _ in range(max(1, int(self._workbench.node_num/32))):
                DhtNetwork.log('Removing all nodes hosting target values...')
                cluster_ops_count = 0
                for proc in self._workbench.procs:
                    DhtNetwork.log('[REMOVE]: sending shutdown request to', proc)
                    proc.sendNodesRequest(
                            DhtNetworkSubProcess.SHUTDOWN_NODE_REQ,
                            DhtFeatureTest.foreignNodes
                    )
                    DhtNetwork.log('sending message stats request')
                    stats = proc.sendGetMessageStats()
                    cluster_ops_count += sum(stats[1:])
                    DhtNetwork.log("3 seconds wait...")
                    time.sleep(3)
                ops_count.append(cluster_ops_count/self._workbench.node_num)

                # checking if values were transfered to new nodes
                foreignNodes_before_delete = DhtFeatureTest.foreignNodes
                DhtNetwork.log('[GET]: trying to fetch persistent values')
                self._dhtGet(consumer, myhash)
                new_nodes = set(DhtFeatureTest.foreignNodes) - set(foreignNodes_before_delete)

                self._result(local_values, new_nodes)

            if self._plot:
                plt.plot(ops_count, color='blue')
                plt.draw()
                plt.ioff()
                plt.show()
        else:
            DhtNetwork.log("[GET]: either couldn't fetch values or nodes hosting values...")
Ejemplo n.º 39
0
    def _multTimeTest(self):
        """
        Multiple put() calls are made from multiple nodes to multiple hashes
        after what a set of 8 nodes is created around each hashes in order to
        enable storage maintenance each nodes. Therefor, this tests will wait 10
        minutes for the nodes to trigger storage maintenance.
        """
        trigger_nodes = []
        bootstrap = self.bootstrap

        N_PRODUCERS = self._num_producers if self._num_values else 16
        DP_TIMEOUT = 1

        hashes = []

        # Generating considerable amount of values of size 1KB.
        VALUE_SIZE = 1024
        NUM_VALUES = self._num_values if self._num_values else 50
        values = [
            Value(''.join(
                random.choice(string.hexdigits)
                for _ in range(VALUE_SIZE)).encode())
            for _ in range(NUM_VALUES)
        ]

        bootstrap.resize(N_PRODUCERS + 2)
        consumer = bootstrap.get(N_PRODUCERS + 1)
        producers = (bootstrap.get(n) for n in range(1, N_PRODUCERS + 1))
        for p in producers:
            hashes.append(random_hash())
            self._dhtPut(p, hashes[-1], *values)

        once = True
        while self._keep_alive or once:
            nodes = set([])
            self._gottaGetThemAllPokeNodes(consumer, hashes, nodes=nodes)

            DhtNetwork.log("Values are found on:")
            for n in nodes:
                DhtNetwork.log(n)

            DhtNetwork.log("Creating 8 nodes around all of these hashes...")
            for _hash in hashes:
                self._trigger_dp(trigger_nodes, _hash, count=8)

            DhtNetwork.log('Waiting', DP_TIMEOUT + 1,
                           'minutes for normal storage maintenance.')
            time.sleep((DP_TIMEOUT + 1) * 60)

            DhtNetwork.log('Deleting old nodes from previous search.')
            for proc in self._workbench.procs:
                DhtNetwork.log('[REMOVE]: sending delete request to', proc)
                proc.sendNodesRequest(DhtNetworkSubProcess.REMOVE_NODE_REQ,
                                      nodes)

            # new consumer (fresh cache)
            bootstrap.resize(N_PRODUCERS + 1)
            bootstrap.resize(N_PRODUCERS + 2)
            consumer = bootstrap.get(N_PRODUCERS + 1)

            nodes_after_time = set([])
            self._gottaGetThemAllPokeNodes(consumer,
                                           hashes,
                                           nodes=nodes_after_time)
            self._result(values, nodes_after_time - nodes)

            once = False
Ejemplo n.º 40
0
    def _multTimeTest(self):
        """
        Multiple put() calls are made from multiple nodes to multiple hashes
        after what a set of 8 nodes is created around each hashes in order to
        enable storage maintenance each nodes. Therefor, this tests will wait 10
        minutes for the nodes to trigger storage maintenance.
        """
        trigger_nodes = []
        bootstrap = self.bootstrap

        N_PRODUCERS = self._num_producers if self._num_values else 16
        DP_TIMEOUT = 1

        hashes = []

        # Generating considerable amount of values of size 1KB.
        VALUE_SIZE = 1024
        NUM_VALUES = self._num_values if self._num_values else 50
        values = [Value(''.join(random.choice(string.hexdigits) for _ in range(VALUE_SIZE)).encode()) for _ in range(NUM_VALUES)]

        bootstrap.resize(N_PRODUCERS+2)
        consumer = bootstrap.get(N_PRODUCERS+1)
        producers = (bootstrap.get(n) for n in range(1,N_PRODUCERS+1))
        for p in producers:
            hashes.append(random_hash())
            self._dhtPut(p, hashes[-1], *values)

        once = True
        while self._keep_alive or once:
            nodes = set([])
            self._gottaGetThemAllPokeNodes(consumer, hashes, nodes=nodes)

            DhtNetwork.log("Values are found on:")
            for n in nodes:
                DhtNetwork.log(n)

            DhtNetwork.log("Creating 8 nodes around all of these hashes...")
            for _hash in hashes:
                self._trigger_dp(trigger_nodes, _hash, count=8)

            DhtNetwork.log('Waiting', DP_TIMEOUT+1, 'minutes for normal storage maintenance.')
            time.sleep((DP_TIMEOUT+1)*60)

            DhtNetwork.log('Deleting old nodes from previous search.')
            for proc in self._workbench.procs:
                DhtNetwork.log('[REMOVE]: sending delete request to', proc)
                proc.sendNodesRequest(
                    DhtNetworkSubProcess.REMOVE_NODE_REQ,
                    nodes)

            # new consumer (fresh cache)
            bootstrap.resize(N_PRODUCERS+1)
            bootstrap.resize(N_PRODUCERS+2)
            consumer = bootstrap.get(N_PRODUCERS+1)

            nodes_after_time = set([])
            self._gottaGetThemAllPokeNodes(consumer, hashes, nodes=nodes_after_time)
            self._result(values, nodes_after_time - nodes)

            once = False
Ejemplo n.º 41
0
 def lookupDoneCb(ok):
     DhtNetwork.log('[LOOKUP]:', PhtTest.key, "--",
                    "success!" if ok else "Fail...")
     with FeatureTest.lock:
         FeatureTest.lock.notify()
Ejemplo n.º 42
0
    def _getsTimesTest(self):
        """
        Tests for performance of the DHT doing multiple get() operation.
        """
        bootstrap = self.bootstrap

        plt.ion()

        fig, axes = plt.subplots(2, 1)
        fig.tight_layout()

        lax = axes[0]
        hax = axes[1]

        lines = None#ax.plot([])
        #plt.ylabel('time (s)')
        hax.set_ylim(0, 2)

        # let the network stabilise
        plt.pause(60)

        #start = time.time()
        times = []

        lock = threading.Condition()
        done = 0

        def getcb(v):
            nonlocal bootstrap
            DhtNetwork.log("found", v)
            return True

        def donecb(ok, nodes):
            nonlocal bootstrap, lock, done, times
            t = time.time()-start
            with lock:
                if not ok:
                    DhtNetwork.log("failed !")
                times.append(t)
                done -= 1
                lock.notify()

        def update_plot():
            nonlocal lines
            while lines:
                l = lines.pop()
                l.remove()
                del l
            lines = plt.plot(times, color='blue')
            plt.draw()

        def run_get():
            nonlocal done
            done += 1
            start = time.time()
            bootstrap.front().get(InfoHash.getRandom(), getcb, lambda ok, nodes: donecb(ok, nodes, start))

        plt.pause(5)

        plt.show()
        update_plot()

        times = []
        for n in range(10):
            self._workbench.replace_cluster()
            plt.pause(2)
            DhtNetwork.log("Getting 50 random hashes succesively.")
            for i in range(50):
                with lock:
                    done += 1
                    start = time.time()
                    bootstrap.front().get(InfoHash.getRandom(), getcb, donecb)
                    while done > 0:
                        lock.wait()
                        update_plot()
                update_plot()
            print("Took", np.sum(times), "mean", np.mean(times), "std", np.std(times), "min", np.min(times), "max", np.max(times))

        print('GET calls timings benchmark test : DONE. '  \
                'Close Matplotlib window for terminating the program.')
        plt.ioff()
        plt.show()
Ejemplo n.º 43
0
 def putDoneCb(ok, nodes):
     if not ok:
         DhtNetwork.log("[PUT]: failed!")
     with FeatureTest.lock:
         FeatureTest.done -= 1
         FeatureTest.lock.notify()
Ejemplo n.º 44
0
 def getcb(value):
     DhtNetwork.log('[GET]: %s' % value)
     DhtFeatureTest.foreignValues.append(value)
     return True
Ejemplo n.º 45
0
    def _deleteTest(self):
        """
        It uses Dht shutdown call from the API to gracefuly finish the nodes one
        after the other.
        """
        bootstrap = self.bootstrap

        ops_count = []

        bootstrap.resize(3)
        consumer = bootstrap.get(1)
        producer = bootstrap.get(2)

        myhash = random_hash()
        local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')]

        self._dhtPut(producer, myhash, *local_values)

        #checking if values were transfered
        self._dhtGet(consumer, myhash)
        if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
            if DhtFeatureTest.foreignValues:
                DhtNetwork.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ',
                        len(local_values), ' values successfully put.')
            else:
                DhtNetwork.log('[GET]: 0 values successfully put')


        if DhtFeatureTest.foreignValues and DhtFeatureTest.foreignNodes:
            DhtNetwork.log('Values are found on :')
            for node in DhtFeatureTest.foreignNodes:
                DhtNetwork.log(node)

            #DhtNetwork.log("Waiting a minute for the network to settle down.")
            #time.sleep(60)

            for _ in range(max(1, int(self._workbench.node_num/32))):
                DhtNetwork.log('Removing all nodes hosting target values...')
                cluster_ops_count = 0
                for proc in self._workbench.procs:
                    DhtNetwork.log('[REMOVE]: sending shutdown request to', proc)
                    proc.sendNodesRequest(
                            DhtNetworkSubProcess.SHUTDOWN_NODE_REQ,
                            DhtFeatureTest.foreignNodes
                    )
                    DhtNetwork.log('sending message stats request')
                    stats = proc.sendGetMessageStats()
                    cluster_ops_count += sum(stats[1:])
                    DhtNetwork.log("5 seconds wait...")
                    time.sleep(5)
                ops_count.append(cluster_ops_count/self._workbench.node_num)

                # checking if values were transfered to new nodes
                foreignNodes_before_delete = DhtFeatureTest.foreignNodes
                DhtNetwork.log('[GET]: trying to fetch persistent values')
                self._dhtGet(consumer, myhash)
                new_nodes = set(DhtFeatureTest.foreignNodes) - set(foreignNodes_before_delete)

                self._result(local_values, new_nodes)

            if self._op_plot:
                display_plot(ops_count, color='blue')
        else:
            DhtNetwork.log("[GET]: either couldn't fetch values or nodes hosting values...")
Ejemplo n.º 46
0
    def _result(self, local_values, new_nodes):
        bootstrap = self.bootstrap
        if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
            DhtNetwork.log('[GET]: Only %s on %s values persisted.' %
                    (len(DhtFeatureTest.foreignValues), len(local_values)))
        else:
            DhtNetwork.log('[GET]: All values successfully persisted.')
        if DhtFeatureTest.foreignValues:
            if new_nodes:
                DhtNetwork.log('Values are newly found on:')
                for node in new_nodes:
                    DhtNetwork.log(node)
                if self._dump_storage:
                    DhtNetwork.log('Dumping all storage log from '\
                                  'hosting nodes.')

                    for proc in self._workbench.procs:
                        proc.sendDumpStorage(DhtFeatureTest.foreignNodes)
            else:
                DhtNetwork.log("Values didn't reach new hosting nodes after shutdown.")