def test_node_load_after_one_node_drops_all_msgs(looper, txnPoolNodeSet, tconf, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) nodes = txnPoolNodeSet x = nodes[-1] with capsys.disabled(): print("Patching node {}".format(x)) def handleOneNodeMsg(self, wrappedMsg): # do nothing with an incoming node message pass x.handleOneNodeMsg = MethodType(handleOneNodeMsg, x) client_batches = 120 txns_per_batch = 25 for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'.format( i + 1, txns_per_batch, perf_counter() - s))
def nodeCreatedAfterSomeTxns(txnPoolNodeSet, tdirWithPoolTxns, poolTxnStewardData, tconf, allPluginsPath, request): with Looper(debug=True) as looper: client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) txnCount = getValueFromModule(request, "txnCount", 5) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txnCount, timeoutPerReq=25) newStewardName = randomString() newNodeName = "Epsilon" newStewardClient, newStewardWallet, newNode = addNewStewardAndNode( looper, client, wallet, newStewardName, newNodeName, tdirWithPoolTxns, tconf, allPluginsPath=allPluginsPath, autoStart=True) yield looper, newNode, client, wallet, newStewardClient, \ newStewardWallet
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnClientData): """ Client should not be able to connect to nodes even after it has changed its HA. Since running on a local environment, only checking change of port. Dont know how to change IP. :return: """ # TODO: Check for change of IP too # name, seed = poolTxnClientData # signer = SimpleSigner(seed=seed) # name = "testClient96541" # ha = genHa() # client = genTestClient(txnPoolNodeSet, signer=signer, ha=ha, # tmpdir=tdirWithPoolTxns, usePoolLedger=True, # name=name) client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet) basedirpath = client.basedirpath looper.removeProdable(client) # Removing RAET keep directory otherwise the client will use the same port # since it will a directory of its name in the keep shutil.rmtree(os.path.join(basedirpath, client.name), ignore_errors=True) ha = genHa() client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId, ha=ha, tmpdir=tdirWithPoolTxns, usePoolLedger=True, name=client.name) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet)
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnClientData): """ Client should not be able to connect to nodes even after it has changed its HA. Since running on a local environment, only checking change of port. Dont know how to change IP. :return: """ # TODO: Check for change of IP too client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet) basedirpath = client.basedirpath looper.removeProdable(client) # Removing RAET keep directory otherwise the client will use the same port # since it will a directory of its name in the keep shutil.rmtree(os.path.join(basedirpath, client.name), ignore_errors=True) ha = genHa() client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId, ha=ha, tmpdir=tdirWithPoolTxns, usePoolLedger=True, name=client.name) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet)
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames, tdirWithPoolTxns, poolTxnData, noRetryReq): """ Client should not be using node registry but pool transaction file :return: """ clients = [] for name in poolTxnClientNames: seed = poolTxnData["seeds"][name].encode() client, wallet = buildPoolClientAndWallet((name, seed), tdirWithPoolTxns) txnPoolNodesLooper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client, *txnPoolNodeSet) clients.append((client, wallet)) n = 250 timeOutPerReq = 3 for (client, wallet) in clients: logger.debug("{} sending {} requests".format(client, n)) sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client, n, 1, timeOutPerReq) logger.debug("{} sent {} requests".format(client, n)) for node in txnPoolNodeSet: logger.debug("{} has requests {} with size {}". format(node, len(node.requests), get_size(node.requests))) for replica in node.replicas: logger.debug("{} has prepares {} with size {}". format(replica, len(replica.prepares), get_size(replica.prepares))) logger.debug("{} has commits {} with size {}". format(replica, len(replica.commits), get_size(replica.commits)))
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet, tdirWithPoolTxns, tdirWithClientPoolTxns, poolTxnClientData): """ Client should not be able to connect to nodes even after it has changed its HA. Since running on a local environment, only checking change of port. Dont know how to change IP. :return: """ # TODO: Check for change of IP too client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithClientPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet) keys_dir = os.path.join(client.keys_dir, client.name) client.stop() looper.removeProdable(client) ha = genHa() client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId, ha=ha, tmpdir=tdirWithClientPoolTxns, usePoolLedger=True, name=client.name) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *txnPoolNodeSet)
def nodeCreatedAfterSomeTxns(looper, testNodeClass, do_post_node_creation, txnPoolNodeSet, tdir, tdirWithClientPoolTxns, poolTxnStewardData, tconf, allPluginsPath, request): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithClientPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) txnCount = getValueFromModule(request, "txnCount", 5) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txnCount) newStewardName = randomString() newNodeName = "Epsilon" newStewardClient, newStewardWallet, newNode = addNewStewardAndNode( looper, client, wallet, newStewardName, newNodeName, tdir, tdirWithClientPoolTxns, tconf, nodeClass=testNodeClass, allPluginsPath=allPluginsPath, autoStart=True, do_post_node_creation=do_post_node_creation) yield looper, newNode, client, wallet, newStewardClient, \ newStewardWallet
def demote_primary_node(looper, initial_pool_of_nodes, pool_of_nodes, poolTxnStewardNames, poolTxnData, tdirWithClientPoolTxns): demoted_node = [node for node in pool_of_nodes if node.has_master_primary][0] indx = initial_pool_of_nodes.index(demoted_node) steward_name = poolTxnStewardNames[indx] stewards_seed = poolTxnData["seeds"][steward_name].encode() stewardClient, stewardWallet = buildPoolClientAndWallet( (steward_name, stewards_seed), tdirWithClientPoolTxns) looper.add(stewardClient) looper.run(stewardClient.ensureConnectedToNodes()) node_data = { ALIAS: demoted_node.name, SERVICES: [] } updateNodeData(looper, stewardClient, stewardWallet, demoted_node, node_data) pool_of_nodes = list(set(pool_of_nodes) - {demoted_node}) return pool_of_nodes
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames, tdirWithPoolTxns, poolTxnData, noRetryReq): """ Client should not be using node registry but pool transaction file :return: """ clients = [] for name in poolTxnClientNames: seed = poolTxnData["seeds"][name].encode() client, wallet = buildPoolClientAndWallet((name, seed), tdirWithPoolTxns) txnPoolNodesLooper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client, *txnPoolNodeSet) clients.append((client, wallet)) n = 250 timeOutPerReq = 3 for (client, wallet) in clients: logger.debug("{} sending {} requests".format(client, n)) sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client, n, 1, timeOutPerReq) logger.debug("{} sent {} requests".format(client, n)) for node in txnPoolNodeSet: logger.debug("{} has requests {} with size {}".format( node, len(node.requests), get_size(node.requests))) for replica in node.replicas: logger.debug("{} has prepares {} with size {}".format( replica, len(replica.prepares), get_size(replica.prepares))) logger.debug("{} has commits {} with size {}".format( replica, len(replica.commits), get_size(replica.commits)))
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames, tdirWithPoolTxns, poolTxnData, noRetryReq): """ Client should not be using node registry but pool transaction file :return: """ clients = [] for name in poolTxnClientNames: seed = poolTxnData["seeds"][name].encode() client, wallet = buildPoolClientAndWallet((name, seed), tdirWithPoolTxns) txnPoolNodesLooper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client, *txnPoolNodeSet) clients.append((client, wallet)) numRequests = 250 fVal = 1 for (client, wallet) in clients: logger.debug("{} sending {} requests".format(client, numRequests)) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, numRequests) logger.debug("{} sent {} requests".format(client, numRequests)) for node in txnPoolNodeSet: logger.debug("{} has requests {} with size {}".format( node, len(node.requests), get_size(node.requests))) for replica in node.replicas.values(): logger.debug("{} has prepares {} with size {}".format( replica, len(replica.prepares), get_size(replica.prepares))) logger.debug("{} has commits {} with size {}".format( replica, len(replica.commits), get_size(replica.commits)))
def clientAndWallet2(looper, poolTxnClientData, tdirWithClientPoolTxns): client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithClientPoolTxns) looper.add(client) looper.run(client.ensureConnectedToNodes()) yield client, wallet client.stop()
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded, poolTxnClientData): newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) # Making the change HA txn an confirming its succeeded changeNodeHa(looper, newSteward, newStewardWallet, newNode, nodeHa=nodeNewHa, clientHa=clientNewHa) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: restartedNode = TestNode(node.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) looper.run( eventually(checkNodeLedgersForEquality, node, *restartedNodes[:-1], retryWait=1, timeout=10)) # Building a new client that reads from the genesis txn file # but is able to connect to all nodes client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *restartedNodes)
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdir, tdirWithPoolTxns, tdirWithClientPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded, poolTxnClientData): newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) # Making the change HA txn an confirming its succeeded op = { ALIAS: newNode.name, NODE_IP: nodeNewHa.host, NODE_PORT: nodeNewHa.port, CLIENT_IP: clientNewHa.host, CLIENT_PORT: clientNewHa.port, } updateNodeData(looper, newSteward, newStewardWallet, newNode, op) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed config_helper = PNodeConfigHelper(newNode.name, tconf, chroot=tdir) node = TestNode(newNode.name, config_helper=config_helper, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) waitNodeDataEquality(looper, node, *restartedNodes[:-1]) # Building a new client that reads from the genesis txn file # but is able to connect to all nodes client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithClientPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *restartedNodes)
def stewardClientAndWallet(poolNodesCreated, looper, tdirWithDomainTxns, poolTxnStewardData): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithDomainTxns, clientClass=TestClient, walletClass=Wallet) client.registerObserver(wallet.handleIncomingReply) looper.add(client) looper.run(client.ensureConnectedToNodes()) makePendingTxnsRequest(client, wallet) return client, wallet
def poolAfterSomeTxns(looper, txnPoolNodesLooper, txnPoolNodeSet, tdirWithClientPoolTxns, poolTxnStewardData, allPluginsPath, request): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithClientPoolTxns, clientClass=TestClient) looper.run(checkNodesConnected(txnPoolNodeSet)) looper.add(client) looper.run(client.ensureConnectedToNodes()) txnCount = getValueFromModule(request, "txnCount", 5) sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client, txnCount) yield looper, client, wallet
def stewardAndWalletForMasterNode(looper, poolTxnData, poolTxnStewardNames, tdirWithClientPoolTxns, txnPoolNodeSet, txnPoolMasterNodes): primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) master_node = txnPoolMasterNodes[0] stewardName = poolTxnStewardNames[primariesIdxs[0]] stewardsSeed = poolTxnData["seeds"][stewardName].encode() stewardClient, stewardWallet = buildPoolClientAndWallet( (stewardName, stewardsSeed), tdirWithClientPoolTxns) looper.add(stewardClient) looper.run(stewardClient.ensureConnectedToNodes()) return stewardClient, stewardWallet
def test_node_load_consistent_time(tconf, change_checkpoint_freq, disable_node_monitor_config, looper, txnPoolNodeSet, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): # One of the reason memory grows is because spylog grows client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) client_batches = 300 txns_per_batch = 25 time_log = [] warm_up_batches = 10 tolerance_factor = 2 from pympler import asizeof for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) t = perf_counter() - s with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'.format( i + 1, txns_per_batch, t)) print('--------Memory Usage details start') for node in txnPoolNodeSet: # print(sys.getsizeof(node)) print('---Node {}-----'.format(node)) # print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1))) print(get_memory_usage(node, True, get_only_non_empty=True)) for r in node.replicas: print('---Replica {}-----'.format(r)) print(get_memory_usage(r, True, get_only_non_empty=True)) print('--------Memory Usage details end') if len(time_log) >= warm_up_batches: m = mean(time_log) sd = tolerance_factor * pstdev(time_log) assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd) time_log.append(t) # Since client checks inbox for sufficient replies, clear inbox so that # it takes constant time to check replies for each batch client.inBox.clear() client.txnLog.reset()
def stewards_and_wallets(looper, txnPoolNodeSet, pool_txn_stewards_data, tdirWithPoolTxns): clients_and_wallets = [] for pool_txn_steward_data in pool_txn_stewards_data: steward_client, steward_wallet = buildPoolClientAndWallet( pool_txn_steward_data, tdirWithPoolTxns) looper.add(steward_client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward_client, *txnPoolNodeSet) clients_and_wallets.append((steward_client, steward_wallet)) yield clients_and_wallets for (client, wallet) in clients_and_wallets: client.stop()
def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) nodes = txnPoolNodeSet x = nodes[-1] with capsys.disabled(): print("Stopping node {} with pool ledger size {}".format( x, x.poolManager.txnSeqNo)) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, x) looper.removeProdable(x) client_batches = 80 txns_per_batch = 10 for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'.format( i + 1, txns_per_batch, perf_counter() - s)) nodeHa, nodeCHa = HA(*x.nodestack.ha), HA(*x.clientstack.ha) newNode = TestNode(x.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(newNode) txnPoolNodeSet[-1] = newNode looper.run(checkNodesConnected(txnPoolNodeSet))
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded, poolTxnClientData): newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) # Making the change HA txn an confirming its succeeded changeNodeHa(looper, newSteward, newStewardWallet, newNode, nodeHa=nodeNewHa, clientHa=clientNewHa) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: restartedNode = TestNode(node.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) looper.run(eventually(checkNodeLedgersForEquality, node, *restartedNodes[:-1], retryWait=1, timeout=10)) # Building a new client that reads from the genesis txn file # but is able to connect to all nodes client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) looper.add(client) ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, *restartedNodes)
def test_node_load(looper, txnPoolNodeSet, tconf, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) client_batches = 150 txns_per_batch = 25 for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, perf_counter() - s))
def checkStewardAdded(looper, poolTxnStewardData, tdirWithPoolTxns): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns) looper.add(client) looper.run(client.ensureConnectedToNodes()) sigseed = b'55555555555555555555555555555555' newSigner = SimpleSigner(sigseed) op = { TXN_TYPE: NYM, ROLE: STEWARD, TARGET_NYM: newSigner.verkey, ALIAS: "Robert", } req = wallet.signOp(op) client.submitReqs(req) def chk(): assert client.getReply(*req.key) == (None, "NOT_FOUND") looper.run(eventually(chk, retryWait=1, timeout=5))
def checkStewardAdded(poolTxnStewardData, tdirWithPoolTxns): with Looper(debug=True) as looper: client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns) looper.add(client) looper.run(client.ensureConnectedToNodes()) sigseed = b'55555555555555555555555555555555' newSigner = SimpleSigner(sigseed) op = { TXN_TYPE: NYM, ROLE: STEWARD, TARGET_NYM: newSigner.verkey, ALIAS: "Robert", } req = wallet.signOp(op) client.submitReqs(req) def chk(): assert client.getReply(req.reqId) == (None, "NOT_FOUND") looper.run(eventually(chk, retryWait=1, timeout=5))
def pre_check(tconf, looper, txnPoolNodeSet, tdirWithClientPoolTxns, poolTxnStewardData): # TODO: Maybe this needs to be extracted in another fixture client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithClientPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) for i in range(tconf.ProcessedBatchMapsToKeep - 1): sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1) # All node maintain the same map from txn range to 3PC looper.run(eventually(chk_if_equal_txn_to_3pc, txnPoolNodeSet)) for i in range(3): sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1) # All node maintain the same map from txn range to 3PC and its equal to # `tconf.ProcessedBatchMapsToKeep` even after sending more batches than # `tconf.ProcessedBatchMapsToKeep`, which shows the garbage cleaning in # action looper.run(eventually(chk_if_equal_txn_to_3pc, txnPoolNodeSet, tconf.ProcessedBatchMapsToKeep))
def nodeCreatedAfterSomeTxns( txnPoolNodesLooper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnStewardData, tconf, allPluginsPath, request ): # with Looper(debug=True) as looper: client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) txnPoolNodesLooper.add(client) txnPoolNodesLooper.run(client.ensureConnectedToNodes()) txnCount = getValueFromModule(request, "txnCount", 5) sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client, txnCount, timeoutPerReq=25) newStewardName = randomString() newNodeName = "Epsilon" newStewardClient, newStewardWallet, newNode = addNewStewardAndNode( txnPoolNodesLooper, client, wallet, newStewardName, newNodeName, tdirWithPoolTxns, tconf, allPluginsPath=allPluginsPath, autoStart=True, ) yield txnPoolNodesLooper, newNode, client, wallet, newStewardClient, newStewardWallet
def test_node_load_consistent_time(tconf, change_checkpoint_freq, disable_node_monitor_config, looper, txnPoolNodeSet, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): # One of the reason memory grows is because spylog grows client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) client_batches = 300 txns_per_batch = 25 time_log = [] warm_up_batches = 10 tolerance_factor = 2 print_detailed_memory_usage = False from pympler import tracker tr = tracker.SummaryTracker() node_methods_to_capture = [ TestNode.executeBatch, TestNode.recordAndPropagate, TestNode.domainDynamicValidation, TestNode.domainRequestApplication ] times = { n.name: {meth.__name__: [] for meth in node_methods_to_capture} for n in txnPoolNodeSet } for node in txnPoolNodeSet: for meth in node_methods_to_capture: meth_name = meth.__name__ patched = timeit(getattr(node, meth_name), times[node.name][meth_name]) setattr(node, meth_name, patched) for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) t = perf_counter() - s with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'.format( i + 1, txns_per_batch, t)) print('--------Memory Usage details start') for node in txnPoolNodeSet: # print(sys.getsizeof(node)) print('---Node {}-----'.format(node)) # print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1))) print( get_memory_usage(node, print_detailed_memory_usage, get_only_non_empty=True)) for r in node.replicas: print('---Replica {}-----'.format(r)) print( get_memory_usage(r, print_detailed_memory_usage, get_only_non_empty=True)) # if i % 3 == 0: # tr.print_diff() print('--------Memory Usage details end') for node in txnPoolNodeSet: for meth in node_methods_to_capture: ts = times[node.name][meth.__name__] print('{} {} {} {}'.format(node, meth.__name__, mean(ts), ts)) if len(time_log) >= warm_up_batches: m = mean(time_log) sd = tolerance_factor * pstdev(time_log) assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd) time_log.append(t) # Since client checks inbox for sufficient replies, clear inbox so that # it takes constant time to check replies for each batch client.inBox.clear() client.txnLog.reset()
def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithPoolTxns): return buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns)
def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData, tdirWithPoolTxns): return buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns)
def new_client(poolTxnClientData, tdirWithPoolTxns): client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) return (client, wallet)
def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithPoolTxns): client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns) yield client, wallet client.stop()
def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData, tdirWithPoolTxns): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns) yield client, wallet client.stop()
def testStewardSuspendsNode(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded, poolTxnStewardData, allPluginsPath): newSteward, newStewardWallet, newNode = nodeThetaAdded newNodeNym = hexToFriendly(newNode.nodestack.local.signer.verhex) suspendNode(looper, newSteward, newStewardWallet, newNodeNym, newNode.name) # Check suspended node does not exist in any nodeReg or remotes of # nodes or clients txnPoolNodeSet = txnPoolNodeSet[:-1] for node in txnPoolNodeSet: looper.run(eventually(checkNodeNotInNodeReg, node, newNode.name)) for client in (steward1, newSteward): looper.run(eventually(checkNodeNotInNodeReg, client, newNode.name)) # Check a client can send request and receive replies req = sendRandomRequest(newStewardWallet, newSteward) checkSufficientRepliesForRequests(looper, newSteward, [ req, ], timeoutPerReq=10) # Check that a restarted client or node does not connect to the suspended # node steward1.stop() looper.removeProdable(steward1) steward1, stewardWallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns) looper.add(steward1) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) looper.run(eventually(checkNodeNotInNodeReg, steward1, newNode.name)) newNode.stop() looper.removeProdable(newNode) # TODO: There is a bug that if a primary node is turned off, it sends # duplicate Pre-Prepare and gets blacklisted. Here is the gist # https://gist.github.com/lovesh/c16989616ebb6856f9fa2905c14dc4b7 oldNodeIdx, oldNode = [(i, n) for i, n in enumerate(txnPoolNodeSet) if not n.hasPrimary][0] oldNode.stop() looper.removeProdable(oldNode) oldNode = TestNode(oldNode.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath) looper.add(oldNode) txnPoolNodeSet[oldNodeIdx] = oldNode looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run(eventually(checkNodeNotInNodeReg, oldNode, newNode.name)) # Check that a node whose suspension is revoked can reconnect to other # nodes and clients can also connect to that node cancelNodeSuspension(looper, newSteward, newStewardWallet, newNodeNym, newNode.name) nodeTheta = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=newNode.nodestack.ha, cliha=newNode.clientstack.ha) looper.add(nodeTheta) txnPoolNodeSet.append(nodeTheta) looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=30)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)