def shutdownNodeRequest(): nonlocal nr_nodes with nr_nodes_cv: nr_nodes -= 1 DhtNetwork.log("Node quitting the DHT.", "(now "+str(nr_nodes)+" nodes on the dht)") nr_nodes_cv.notify() random.choice(wb.procs).sendNodesRequest(DhtNetworkSubProcess.SHUTDOWN_NODE_REQ)
def newNodeRequest(): nonlocal nr_nodes with nr_nodes_cv: nr_nodes += 1 DhtNetwork.log("Node joining the DHT.", "(now "+str(nr_nodes)+" nodes on the dht)") nr_nodes_cv.notify() random.choice(wb.procs).sendNodesRequest(DhtNetworkSubProcess.NEW_NODE_REQ)
def putRequest(): nonlocal hashes, values, nr_values nr_values += 1 DhtNetwork.log("Random value put on the DHT.", "(now " + str(nr_values) + " values on the dht)") random.choice(wb.procs).sendNodePutRequest( random.choice(hashes).toString(), random.choice(values))
def _trigger_dp(self, trigger_nodes, _hash, count=1): """ Triggers the data persistence over time. In order to this, `count` nodes are created with an id around the hash of a value. @param trigger_nodes: List of created nodes. The nodes created in this function are append to this list. @type trigger_nodes: list @param _hash: Is the id of the value around which creating nodes. @type _hash: InfoHash @param count: The number of nodes to create with id around the id of value. @type count: int """ _hash_str = _hash.toString().decode() _hash_int = int(_hash_str, 16) for i in range(int(-count/2), int(count/2)+1): _hash_str = '{:40x}'.format(_hash_int + i) config = DhtConfig() config.setNodeId(InfoHash(_hash_str.encode())) n = DhtRunner() n.run(config=config) n.bootstrap(self._bootstrap.ip4, str(self._bootstrap.port)) DhtNetwork.log('Node','['+_hash_str+']', 'started around', _hash.toString().decode() if n.isRunning() else 'failed to start...' ) trigger_nodes.append(n)
def _trigger_dp(self, trigger_nodes, _hash, count=1): """ Triggers the data persistence over time. In order to this, `count` nodes are created with an id around the hash of a value. @param trigger_nodes: List of created nodes. The nodes created in this function are append to this list. @type trigger_nodes: list @param _hash: Is the id of the value around which creating nodes. @type _hash: InfoHash @param count: The number of nodes to create with id around the id of value. @type count: int """ _hash_str = _hash.toString().decode() _hash_int = int(_hash_str, 16) for i in range(int(-count / 2), int(count / 2) + 1): _hash_str = '{:40x}'.format(_hash_int + i) config = DhtConfig() config.setNodeId(InfoHash(_hash_str.encode())) n = DhtRunner() n.run(config=config) n.bootstrap(self._bootstrap.ip4, str(self._bootstrap.port)) DhtNetwork.log( 'Node', '[' + _hash_str + ']', 'started around', _hash.toString().decode() if n.isRunning() else 'failed to start...') trigger_nodes.append(n)
def _dhtPut(self, producer, _hash, *values): for val in values: with FeatureTest.lock: DhtNetwork.log('[PUT]: %s' % val) FeatureTest.done += 1 producer.put(_hash, val, DhtFeatureTest.putDoneCb) while FeatureTest.done > 0: FeatureTest.lock.wait()
def shutdownNodeRequest(): nonlocal nr_nodes with nr_nodes_cv: nr_nodes -= 1 DhtNetwork.log("Node quitting the DHT.", "(now " + str(nr_nodes) + " nodes on the dht)") nr_nodes_cv.notify() random.choice(wb.procs).sendNodesRequest( DhtNetworkSubProcess.SHUTDOWN_NODE_REQ)
def newNodeRequest(): nonlocal nr_nodes with nr_nodes_cv: nr_nodes += 1 DhtNetwork.log("Node joining the DHT.", "(now " + str(nr_nodes) + " nodes on the dht)") nr_nodes_cv.notify() random.choice(wb.procs).sendNodesRequest( DhtNetworkSubProcess.NEW_NODE_REQ)
def _dhtGet(self, consumer, _hash): DhtFeatureTest.foreignValues = [] DhtFeatureTest.foreignNodes = [] with FeatureTest.lock: FeatureTest.done += 1 DhtNetwork.log('[GET]:', _hash.toString()) consumer.get(_hash, DhtFeatureTest.getcb, DhtFeatureTest.getDoneCb) while FeatureTest.done > 0: FeatureTest.lock.wait()
def _dhtGet(self, consumer, _hash): DhtFeatureTest.foreignValues = [] DhtFeatureTest.foreignNodes = [] with FeatureTest.lock: FeatureTest.done += 1 while FeatureTest.done > 0: DhtNetwork.log('[GET]:', _hash.toString()) consumer.get(_hash, DhtFeatureTest.getcb, DhtFeatureTest.getDoneCb) FeatureTest.lock.wait()
def _dhtPut(self, producer, _hash, *values): for val in values: with FeatureTest.lock: vstr = val.__str__()[:100] DhtNetwork.log('[PUT]: %s' % vstr + ("..." if len(vstr) > 100 else "")) FeatureTest.done += 1 while FeatureTest.done > 0: producer.put(_hash, val, DhtFeatureTest.putDoneCb) FeatureTest.lock.wait()
def donecb(ok, nodes): nonlocal bootstrap, lock, done, times t = time.time()-start with lock: if not ok: DhtNetwork.log("failed !") times.append(t) done -= 1 lock.notify()
def getDoneCb(ok, nodes): with FeatureTest.lock: if not ok: DhtNetwork.log("[GET]: failed!") else: for node in nodes: if not node.getNode().isExpired(): DhtFeatureTest.foreignNodes.append(node.getId().toString()) FeatureTest.done -= 1 FeatureTest.lock.notify()
def _dhtPut(self, producer, _hash, *values): with FeatureTest.lock: for val in values: vstr = val.__str__()[:100] DhtNetwork.log('[PUT]:', _hash.toString(), '->', vstr + ("..." if len(vstr) > 100 else "")) FeatureTest.done += 1 producer.put(_hash, val, DhtFeatureTest.putDoneCb) while FeatureTest.done > 0: FeatureTest.lock.wait()
def _insertTest(self): """TODO: Docstring for _massIndexTest. """ bootstrap = self._bootstrap bootstrap.resize(2) dht = bootstrap.get(1) NUM_DIG = max(math.log(self._num_keys, 2) / 4, 5) # at least 5 digit keys. keyspec = collections.OrderedDict([('foo', NUM_DIG)]) pht = Pht(b'foo_index', keyspec, dht) DhtNetwork.log('PHT has', pht.MAX_NODE_ENTRY_COUNT, 'node' + ('s' if pht.MAX_NODE_ENTRY_COUNT > 1 else ''), 'per leaf bucket.') keys = [{ [_ for _ in keyspec.keys()][0]: ''.join(random.SystemRandom().choice(string.hexdigits) for _ in range(NUM_DIG)).encode() } for n in range(self._num_keys)] all_entries = {} # Index all entries. for key in keys: PhtTest.key = key with FeatureTest.lock: time_taken = timer(pht.insert, key, IndexValue(random_hash()), PhtTest.insertDoneCb) if self._timer: DhtNetwork.log('This insert step took : ', time_taken, 'second') FeatureTest.lock.wait() time.sleep(1) # Recover entries now that the trie is complete. for key in keys: PhtTest.key = key with FeatureTest.lock: time_taken = timer(pht.lookup, key, PhtTest.lookupCb, PhtTest.lookupDoneCb) if self._timer: DhtNetwork.log('This lookup step took : ', time_taken, 'second') FeatureTest.lock.wait() all_entries[PhtTest.prefix] = [ e.__str__() for e in PhtTest.indexEntries ] for p in all_entries.keys(): DhtNetwork.log('All entries under prefix', p, ':') DhtNetwork.log(all_entries[p]) PhtTest.drawTrie(all_entries)
def _insertTest(self): """TODO: Docstring for _massIndexTest. """ bootstrap = self._bootstrap bootstrap.resize(2) dht = bootstrap.get(1) NUM_DIG = max(math.log(self._num_keys, 2)/4, 5) # at least 5 digit keys. keyspec = collections.OrderedDict([('foo', NUM_DIG)]) pht = Pht(b'foo_index', keyspec, dht) DhtNetwork.log('PHT has', pht.MAX_NODE_ENTRY_COUNT, 'node'+ ('s' if pht.MAX_NODE_ENTRY_COUNT > 1 else ''), 'per leaf bucket.') keys = [{ [_ for _ in keyspec.keys()][0] : ''.join(random.SystemRandom().choice(string.hexdigits) for _ in range(NUM_DIG)).encode() } for n in range(self._num_keys)] all_entries = {} # Index all entries. for key in keys: PhtTest.key = key with FeatureTest.lock: time_taken = timer(pht.insert, key, IndexValue(random_hash()), PhtTest.insertDoneCb) if self._timer: DhtNetwork.log('This insert step took : ', time_taken, 'second') FeatureTest.lock.wait() time.sleep(1) # Recover entries now that the trie is complete. for key in keys: PhtTest.key = key with FeatureTest.lock: time_taken = timer(pht.lookup, key, PhtTest.lookupCb, PhtTest.lookupDoneCb) if self._timer: DhtNetwork.log('This lookup step took : ', time_taken, 'second') FeatureTest.lock.wait() all_entries[PhtTest.prefix] = [e.__str__() for e in PhtTest.indexEntries] for p in all_entries.keys(): DhtNetwork.log('All entries under prefix', p, ':') DhtNetwork.log(all_entries[p]) PhtTest.drawTrie(all_entries)
def lookupCb(vals, prefix): PhtTest.indexEntries = list(vals) PhtTest.prefix = prefix.decode() DhtNetwork.log('Index name: <todo>') DhtNetwork.log('Leaf prefix:', prefix) for v in vals: DhtNetwork.log('[ENTRY]:', v)
def _replaceClusterTest(self): """ It replaces all clusters one after the other. """ clusters = 8 bootstrap = self.bootstrap bootstrap.resize(3) consumer = bootstrap.get(1) producer = bootstrap.get(2) myhash = random_hash() local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')] self._dhtPut(producer, myhash, *local_values) self._dhtGet(consumer, myhash) initial_nodes = DhtFeatureTest.foreignNodes DhtNetwork.log('Replacing', clusters, 'random clusters successively...') for n in range(clusters): i = random.randint(0, len(self._workbench.procs)-1) proc = self._workbench.procs[i] DhtNetwork.log('Replacing', proc) proc.sendNodesRequest(DhtNetworkSubProcess.SHUTDOWN_CLUSTER_REQ) self._workbench.stop_cluster(i) self._workbench.start_cluster(i) DhtNetwork.log('[GET]: trying to fetch persistent values') self._dhtGet(consumer, myhash) new_nodes = set(DhtFeatureTest.foreignNodes) - set(initial_nodes) self._result(local_values, new_nodes)
def _replaceClusterTest(self): """ It replaces all clusters one after the other. """ #clusters = opts['clusters'] if 'clusters' in opts else 5 clusters = 5 bootstrap = self.bootstrap bootstrap.resize(3) consumer = bootstrap.get(1) producer = bootstrap.get(2) myhash = random_hash() local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')] self._dhtPut(producer, myhash, *local_values) self._dhtGet(consumer, myhash) initial_nodes = DhtFeatureTest.foreignNodes DhtNetwork.log('Replacing', clusters, 'random clusters successively...') for n in range(clusters): i = random.randint(0, len(self._workbench.procs)-1) proc = self._workbench.procs[i] DhtNetwork.log('Replacing', proc) proc.sendShutdown() self._workbench.stop_cluster(i) self._workbench.start_cluster(i) DhtNetwork.log('[GET]: trying to fetch persistent values') self._dhtGet(consumer, myhash) new_nodes = set(DhtFeatureTest.foreignNodes) - set(initial_nodes) self._result(local_values, new_nodes)
def _delete(self): """ Tests for performance of get() and put() operations on the network while deleting around the target hash. """ bootstrap = self.bootstrap bootstrap.resize(3) consumer = bootstrap.get(1) producer = bootstrap.get(2) myhash = random_hash() local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')] for _ in range(max(1, int(self._workbench.node_num/32))): self._dhtGet(consumer, myhash) DhtNetwork.log("Waiting 15 seconds...") time.sleep(15) self._dhtPut(producer, myhash, *local_values) #checking if values were transfered self._dhtGet(consumer, myhash) DhtNetwork.log('Values are found on :') for node in DhtFeatureTest.foreignNodes: DhtNetwork.log(node) if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues): if DhtFeatureTest.foreignValues: DhtNetwork.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ', len(local_values), ' values successfully put.') else: DhtNetwork.log('[GET]: 0 values successfully put') DhtNetwork.log('Removing all nodes hosting target values...') for proc in self._workbench.procs: DhtNetwork.log('[REMOVE]: sending shutdown request to', proc) proc.sendNodesRequest( DhtNetworkSubProcess.SHUTDOWN_NODE_REQ, DhtFeatureTest.foreignNodes )
def _deleteTest(self): """ It uses Dht shutdown call from the API to gracefuly finish the nodes one after the other. """ bootstrap = self.bootstrap ops_count = [] bootstrap.resize(3) consumer = bootstrap.get(1) producer = bootstrap.get(2) myhash = random_hash() local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')] self._dhtPut(producer, myhash, *local_values) #checking if values were transfered self._dhtGet(consumer, myhash) if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues): if DhtFeatureTest.foreignValues: DhtNetwork.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ', len(local_values), ' values successfully put.') else: DhtNetwork.log('[GET]: 0 values successfully put') if DhtFeatureTest.foreignValues and DhtFeatureTest.foreignNodes: DhtNetwork.log('Values are found on :') for node in DhtFeatureTest.foreignNodes: DhtNetwork.log(node) #DhtNetwork.log("Waiting a minute for the network to settle down.") #time.sleep(60) for _ in range(max(1, int(self._workbench.node_num/32))): DhtNetwork.log('Removing all nodes hosting target values...') cluster_ops_count = 0 for proc in self._workbench.procs: DhtNetwork.log('[REMOVE]: sending shutdown request to', proc) proc.sendNodesRequest( DhtNetworkSubProcess.SHUTDOWN_NODE_REQ, DhtFeatureTest.foreignNodes ) DhtNetwork.log('sending message stats request') stats = proc.sendGetMessageStats() cluster_ops_count += sum(stats[1:]) DhtNetwork.log("3 seconds wait...") time.sleep(3) ops_count.append(cluster_ops_count/self._workbench.node_num) # checking if values were transfered to new nodes foreignNodes_before_delete = DhtFeatureTest.foreignNodes DhtNetwork.log('[GET]: trying to fetch persistent values') self._dhtGet(consumer, myhash) new_nodes = set(DhtFeatureTest.foreignNodes) - set(foreignNodes_before_delete) self._result(local_values, new_nodes) if self._plot: plt.plot(ops_count, color='blue') plt.draw() plt.ioff() plt.show() else: DhtNetwork.log("[GET]: either couldn't fetch values or nodes hosting values...")
def getcb(v): nonlocal bootstrap DhtNetwork.log("found", v) return True
def _getsTimesTest(self): """ Tests for performance of the DHT doing multiple get() operation. """ bootstrap = self.bootstrap plt.ion() fig, axes = plt.subplots(2, 1) fig.tight_layout() lax = axes[0] hax = axes[1] lines = None#ax.plot([]) #plt.ylabel('time (s)') hax.set_ylim(0, 2) # let the network stabilise plt.pause(60) #start = time.time() times = [] lock = threading.Condition() done = 0 def getcb(v): nonlocal bootstrap DhtNetwork.log("found", v) return True def donecb(ok, nodes): nonlocal bootstrap, lock, done, times t = time.time()-start with lock: if not ok: DhtNetwork.log("failed !") times.append(t) done -= 1 lock.notify() def update_plot(): nonlocal lines while lines: l = lines.pop() l.remove() del l lines = plt.plot(times, color='blue') plt.draw() def run_get(): nonlocal done done += 1 start = time.time() bootstrap.front().get(InfoHash.getRandom(), getcb, lambda ok, nodes: donecb(ok, nodes, start)) plt.pause(5) plt.show() update_plot() times = [] for n in range(10): self._workbench.replace_cluster() plt.pause(2) DhtNetwork.log("Getting 50 random hashes succesively.") for i in range(50): with lock: done += 1 start = time.time() bootstrap.front().get(InfoHash.getRandom(), getcb, donecb) while done > 0: lock.wait() update_plot() update_plot() print("Took", np.sum(times), "mean", np.mean(times), "std", np.std(times), "min", np.min(times), "max", np.max(times)) print('GET calls timings benchmark test : DONE. ' \ 'Close Matplotlib window for terminating the program.') plt.ioff() plt.show()
def putDoneCb(ok, nodes): if not ok: DhtNetwork.log("[PUT]: failed!") with FeatureTest.lock: FeatureTest.done -= 1 FeatureTest.lock.notify()
def _multTimeTest(self): """ Multiple put() calls are made from multiple nodes to multiple hashes after what a set of 8 nodes is created around each hashes in order to enable storage maintenance each nodes. Therefor, this tests will wait 10 minutes for the nodes to trigger storage maintenance. """ bootstrap = self.bootstrap N_PRODUCERS = 16 hashes = [] values = [Value(b'foo')] nodes = set([]) # prevents garbage collecting of unused flood nodes during the test. flood_nodes = [] def gottaGetThemAllPokeNodes(nodes=None): nonlocal consumer, hashes for h in hashes: self._dhtGet(consumer, h) if nodes is not None: for n in DhtFeatureTest.foreignNodes: nodes.add(n) def createNodesAroundHash(_hash, radius=4): nonlocal flood_nodes _hash_str = _hash.toString().decode() _hash_int = int(_hash_str, 16) for i in range(-radius, radius+1): _hash_str = '{:40x}'.format(_hash_int + i) config = DhtConfig() config.setNodeId(InfoHash(_hash_str.encode())) n = DhtRunner() n.run(config=config) n.bootstrap(self.bootstrap.ip4, str(self.bootstrap.port)) flood_nodes.append(n) bootstrap.resize(N_PRODUCERS+2) consumer = bootstrap.get(1) producers = (bootstrap.get(n) for n in range(2,N_PRODUCERS+2)) for p in producers: hashes.append(random_hash()) self._dhtPut(p, hashes[-1], *values) gottaGetThemAllPokeNodes(nodes=nodes) DhtNetwork.log("Values are found on:") for n in nodes: DhtNetwork.log(n) DhtNetwork.log("Creating 8 nodes around all of these nodes...") for _hash in hashes: createNodesAroundHash(_hash) DhtNetwork.log('Waiting 10 minutes for normal storage maintenance.') time.sleep(10*60) DhtNetwork.log('Deleting old nodes from previous search.') for proc in self._workbench.procs: DhtNetwork.log('[REMOVE]: sending shutdown request to', proc) proc.sendNodesRequest( DhtNetworkSubProcess.REMOVE_NODE_REQ, nodes ) # new consumer (fresh cache) bootstrap.resize(N_PRODUCERS+3) consumer = bootstrap.get(N_PRODUCERS+2) nodes_after_time = set([]) gottaGetThemAllPokeNodes(nodes=nodes_after_time) self._result(values, nodes_after_time - nodes)
def insertDoneCb(ok): DhtNetwork.log('[INSERT]:', PhtTest.key, "--", "success!" if ok else "Fail...") with FeatureTest.lock: FeatureTest.lock.notify()
def _result(self, local_values, new_nodes): bootstrap = self.bootstrap if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues): DhtNetwork.log('[GET]: Only %s on %s values persisted.' % (len(DhtFeatureTest.foreignValues), len(local_values))) else: DhtNetwork.log('[GET]: All values successfully persisted.') if DhtFeatureTest.foreignValues: if new_nodes: DhtNetwork.log('Values are newly found on:') for node in new_nodes: DhtNetwork.log(node) if self._dump_storage: DhtNetwork.log('Dumping all storage log from '\ 'hosting nodes.') for proc in self._workbench.procs: proc.sendDumpStorage(DhtFeatureTest.foreignNodes) else: DhtNetwork.log("Values didn't reach new hosting nodes after shutdown.")
def putRequest(): nonlocal hashes, values, nr_values nr_values += 1 DhtNetwork.log("Random value put on the DHT.", "(now "+ str(nr_values)+" values on the dht)") random.choice(wb.procs).sendNodePutRequest(random.choice(hashes).toString(), random.choice(values))
def _result(self, local_values, new_nodes): bootstrap = self.bootstrap if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues): DhtNetwork.log('[GET]: Only %s on %s values persisted.' % (len(DhtFeatureTest.foreignValues), len(local_values))) else: DhtNetwork.log('[GET]: All values successfully persisted.') if DhtFeatureTest.foreignValues: if new_nodes: DhtNetwork.log('Values are newly found on:') for node in new_nodes: DhtNetwork.log(node) if self._dump_storage: DhtNetwork.log('Dumping all storage log from '\ 'hosting nodes.') for proc in self._workbench.procs: proc.sendNodesRequest(DhtNetworkSubProcess.DUMP_STORAGE_REQ, DhtFeatureTest.foreignNodes) else: DhtNetwork.log("Values didn't reach new hosting nodes after shutdown.")
def getcb(value): vstr = value.__str__()[:100] DhtNetwork.log('[GET]: %s' % vstr + ("..." if len(vstr) > 100 else "")) DhtFeatureTest.foreignValues.append(value) return True
def lookupDoneCb(ok): DhtNetwork.log('[LOOKUP]:', PhtTest.key, "--", "success!" if ok else "Fail...") with FeatureTest.lock: FeatureTest.lock.notify()
def getcb(value): DhtNetwork.log('[GET]: %s' % value) DhtFeatureTest.foreignValues.append(value) return True
def _deleteTest(self): """ It uses Dht shutdown call from the API to gracefuly finish the nodes one after the other. """ bootstrap = self.bootstrap ops_count = [] bootstrap.resize(3) consumer = bootstrap.get(1) producer = bootstrap.get(2) myhash = random_hash() local_values = [Value(b'foo'), Value(b'bar'), Value(b'foobar')] self._dhtPut(producer, myhash, *local_values) #checking if values were transfered self._dhtGet(consumer, myhash) if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues): if DhtFeatureTest.foreignValues: DhtNetwork.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ', len(local_values), ' values successfully put.') else: DhtNetwork.log('[GET]: 0 values successfully put') if DhtFeatureTest.foreignValues and DhtFeatureTest.foreignNodes: DhtNetwork.log('Values are found on :') for node in DhtFeatureTest.foreignNodes: DhtNetwork.log(node) #DhtNetwork.log("Waiting a minute for the network to settle down.") #time.sleep(60) for _ in range(max(1, int(self._workbench.node_num/32))): DhtNetwork.log('Removing all nodes hosting target values...') cluster_ops_count = 0 for proc in self._workbench.procs: DhtNetwork.log('[REMOVE]: sending shutdown request to', proc) proc.sendNodesRequest( DhtNetworkSubProcess.SHUTDOWN_NODE_REQ, DhtFeatureTest.foreignNodes ) DhtNetwork.log('sending message stats request') stats = proc.sendGetMessageStats() cluster_ops_count += sum(stats[1:]) DhtNetwork.log("5 seconds wait...") time.sleep(5) ops_count.append(cluster_ops_count/self._workbench.node_num) # checking if values were transfered to new nodes foreignNodes_before_delete = DhtFeatureTest.foreignNodes DhtNetwork.log('[GET]: trying to fetch persistent values') self._dhtGet(consumer, myhash) new_nodes = set(DhtFeatureTest.foreignNodes) - set(foreignNodes_before_delete) self._result(local_values, new_nodes) if self._op_plot: display_plot(ops_count, color='blue') else: DhtNetwork.log("[GET]: either couldn't fetch values or nodes hosting values...")
def _multTimeTest(self): """ Multiple put() calls are made from multiple nodes to multiple hashes after what a set of 8 nodes is created around each hashes in order to enable storage maintenance each nodes. Therefor, this tests will wait 10 minutes for the nodes to trigger storage maintenance. """ trigger_nodes = [] bootstrap = self.bootstrap N_PRODUCERS = self._num_producers if self._num_values else 16 DP_TIMEOUT = 1 hashes = [] # Generating considerable amount of values of size 1KB. VALUE_SIZE = 1024 NUM_VALUES = self._num_values if self._num_values else 50 values = [Value(''.join(random.choice(string.hexdigits) for _ in range(VALUE_SIZE)).encode()) for _ in range(NUM_VALUES)] bootstrap.resize(N_PRODUCERS+2) consumer = bootstrap.get(N_PRODUCERS+1) producers = (bootstrap.get(n) for n in range(1,N_PRODUCERS+1)) for p in producers: hashes.append(random_hash()) self._dhtPut(p, hashes[-1], *values) once = True while self._keep_alive or once: nodes = set([]) self._gottaGetThemAllPokeNodes(consumer, hashes, nodes=nodes) DhtNetwork.log("Values are found on:") for n in nodes: DhtNetwork.log(n) DhtNetwork.log("Creating 8 nodes around all of these hashes...") for _hash in hashes: self._trigger_dp(trigger_nodes, _hash, count=8) DhtNetwork.log('Waiting', DP_TIMEOUT+1, 'minutes for normal storage maintenance.') time.sleep((DP_TIMEOUT+1)*60) DhtNetwork.log('Deleting old nodes from previous search.') for proc in self._workbench.procs: DhtNetwork.log('[REMOVE]: sending delete request to', proc) proc.sendNodesRequest( DhtNetworkSubProcess.REMOVE_NODE_REQ, nodes) # new consumer (fresh cache) bootstrap.resize(N_PRODUCERS+1) bootstrap.resize(N_PRODUCERS+2) consumer = bootstrap.get(N_PRODUCERS+1) nodes_after_time = set([]) self._gottaGetThemAllPokeNodes(consumer, hashes, nodes=nodes_after_time) self._result(values, nodes_after_time - nodes) once = False
def _multTimeTest(self): """ Multiple put() calls are made from multiple nodes to multiple hashes after what a set of 8 nodes is created around each hashes in order to enable storage maintenance each nodes. Therefor, this tests will wait 10 minutes for the nodes to trigger storage maintenance. """ trigger_nodes = [] bootstrap = self.bootstrap N_PRODUCERS = self._num_producers if self._num_values else 16 DP_TIMEOUT = 1 hashes = [] # Generating considerable amount of values of size 1KB. VALUE_SIZE = 1024 NUM_VALUES = self._num_values if self._num_values else 50 values = [ Value(''.join( random.choice(string.hexdigits) for _ in range(VALUE_SIZE)).encode()) for _ in range(NUM_VALUES) ] bootstrap.resize(N_PRODUCERS + 2) consumer = bootstrap.get(N_PRODUCERS + 1) producers = (bootstrap.get(n) for n in range(1, N_PRODUCERS + 1)) for p in producers: hashes.append(random_hash()) self._dhtPut(p, hashes[-1], *values) once = True while self._keep_alive or once: nodes = set([]) self._gottaGetThemAllPokeNodes(consumer, hashes, nodes=nodes) DhtNetwork.log("Values are found on:") for n in nodes: DhtNetwork.log(n) DhtNetwork.log("Creating 8 nodes around all of these hashes...") for _hash in hashes: self._trigger_dp(trigger_nodes, _hash, count=8) DhtNetwork.log('Waiting', DP_TIMEOUT + 1, 'minutes for normal storage maintenance.') time.sleep((DP_TIMEOUT + 1) * 60) DhtNetwork.log('Deleting old nodes from previous search.') for proc in self._workbench.procs: DhtNetwork.log('[REMOVE]: sending delete request to', proc) proc.sendNodesRequest(DhtNetworkSubProcess.REMOVE_NODE_REQ, nodes) # new consumer (fresh cache) bootstrap.resize(N_PRODUCERS + 1) bootstrap.resize(N_PRODUCERS + 2) consumer = bootstrap.get(N_PRODUCERS + 1) nodes_after_time = set([]) self._gottaGetThemAllPokeNodes(consumer, hashes, nodes=nodes_after_time) self._result(values, nodes_after_time - nodes) once = False