def test_state_regenerated_from_ledger(looper, txnPoolNodeSet, client1, wallet1, client1Connected, tconf, tdirWithPoolTxns, allPluginsPath): """ Node loses its state database but recreates it from ledger after start """ sent_batches = 10 send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 5 * sent_batches, sent_batches) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) node_to_stop = txnPoolNodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv.db_path nodeHa, nodeCHa = HA(*node_to_stop.nodestack.ha), HA( *node_to_stop.clientstack.ha) node_to_stop.stop() looper.removeProdable(node_to_stop) shutil.rmtree(state_db_path) restarted_node = TestNode(node_to_stop.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(restarted_node) txnPoolNodeSet[-1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, restarted_node, *txnPoolNodeSet[:-1])
def testZStackNodeReconnection(tconf, looper, txnPoolNodeSet, client1, wallet1, tdirWithPoolTxns, client1Connected): sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1) npr = [n for n in txnPoolNodeSet if not n.hasPrimary] nodeToCrash = npr[0] idxToCrash = txnPoolNodeSet.index(nodeToCrash) otherNodes = [_ for _ in txnPoolNodeSet if _ != nodeToCrash] def checkFlakyConnected(conn=True): for node in otherNodes: if conn: assert nodeToCrash.nodestack.name in node.nodestack.connecteds else: assert nodeToCrash.nodestack.name not in node.nodestack.connecteds checkFlakyConnected(True) nodeToCrash.stop() looper.removeProdable(nodeToCrash) looper.runFor(1) looper.run(eventually(checkFlakyConnected, False, retryWait=1, timeout=35)) looper.runFor(1) node = TestNode(nodeToCrash.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeToCrash.nodestack.ha, cliha=nodeToCrash.clientstack.ha) looper.add(node) txnPoolNodeSet[idxToCrash] = node looper.run(eventually(checkFlakyConnected, True, retryWait=2, timeout=50)) ensureElectionsDone(looper, txnPoolNodeSet, retryWait=2, timeout=50) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1) checkNodesSendingCommits(txnPoolNodeSet)
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward): new_steward_wallet, new_node = \ sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, 'AnotherSteward' + randomString(4), 'AnotherNode' + randomString(4), tdir, tconf) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) node_new_ha, client_new_ha = genHa(2) logger.debug("{} changing HAs to {} {}".format(new_node, node_new_ha, client_new_ha)) # Making the change HA txn an confirming its succeeded node_dest = hexToFriendly(new_node.nodestack.verhex) sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, node_dest, new_node.name, node_new_ha.host, node_new_ha.port, client_new_ha.host, client_new_ha.port) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir) node = TestNode(new_node.name, config_helper=config_helper, config=tconf, ha=node_new_ha, cliha=client_new_ha) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) waitNodeDataEquality(looper, node, *restartedNodes[:-1]) sdk_pool_refresh(looper, sdk_pool_handle) sdk_ensure_pool_functional(looper, restartedNodes, sdk_wallet_client, sdk_pool_handle)
def clientAndWallet2(looper, poolTxnClientData, tdirWithClientPoolTxns): client, wallet = buildPoolClientAndWallet(poolTxnClientData, tdirWithClientPoolTxns) looper.add(client) looper.run(client.ensureConnectedToNodes()) yield client, wallet client.stop()
def test_node_load_after_add_then_disconnect(newNodeCaughtUp, txnPoolNodeSet, tconf, looper, client1, wallet1, client1Connected, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): """ A node that restarts after some transactions should eventually get the transactions which happened while it was down :return: """ new_node = newNodeCaughtUp with capsys.disabled(): print("Stopping node {} with pool ledger size {}". format(new_node, new_node.poolManager.txnSeqNo)) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node) looper.removeProdable(new_node) client_batches = 80 txns_per_batch = 10 for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, txns_per_batch, override_timeout_limit=True) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, perf_counter() - s)) with capsys.disabled(): print("Starting the stopped node, {}".format(new_node)) nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha) new_node = TestNode( new_node.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(new_node) txnPoolNodeSet[-1] = new_node # Delay catchup reply processing so LedgerState does not change delay_catchup_reply = 5 new_node.nodeIbStasher.delay(cr_delay(delay_catchup_reply)) looper.run(checkNodesConnected(txnPoolNodeSet)) # Make sure ledger starts syncing (sufficient consistency proofs received) looper.run(eventually(check_ledger_state, new_node, DOMAIN_LEDGER_ID, LedgerState.syncing, retryWait=.5, timeout=5)) # Not accurate timeout but a conservative one timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \ 2 * delay_catchup_reply waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4], customTimeout=timeout) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4])
def test_primary_selection_after_primary_demotion_and_pool_restart( looper, txnPoolNodeSet, stewardAndWalletForMasterNode, txnPoolMasterNodes, tdir, tconf): """ Demote primary and restart the pool. Pool should select new primary and have viewNo=0 after restart. """ logger.info( "1. turn off the node which has primary replica for master instanse") master_node = txnPoolMasterNodes[0] client, wallet = stewardAndWalletForMasterNode node_data = {ALIAS: master_node.name, SERVICES: []} updateNodeData(looper, client, wallet, master_node, node_data) restNodes = [ node for node in txnPoolNodeSet if node.name != master_node.name ] ensureElectionsDone(looper, restNodes) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) logger.info("2. restart pool") # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) restNodes = [ node for node in restartedNodes if node.name != master_node.name ] looper.run(checkNodesConnected(restNodes)) ensureElectionsDone(looper, restNodes) checkViewNoForNodes(restNodes, 0) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) primariesIdxs = getPrimaryNodesIdxs(restNodes) assert restNodes[primariesIdxs[0]].name != master_node.name
def test_client_can_not_send_write_requests_until_catchup( looper, poolTxnClientData, tdirWithPoolTxns, txnPoolNodeSet): slow_catch_up(txnPoolNodeSet, 60) client, _ = new_client(poolTxnClientData, tdirWithPoolTxns) looper.add(client) with pytest.raises(TestException): looper.run(eventually(can_send_write_requests, client, timeout=4)) cancel_slow_catch_up(txnPoolNodeSet) looper.run(eventually(can_send_write_requests, client))
def stewardAndWalletForMasterNode(looper, poolTxnData, poolTxnStewardNames, tdirWithPoolTxns, txnPoolNodeSet, txnPoolMasterNodes): primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) master_node = txnPoolMasterNodes[0] stewardName = poolTxnStewardNames[primariesIdxs[0]] stewardsSeed = poolTxnData["seeds"][stewardName].encode() stewardClient, stewardWallet = buildPoolClientAndWallet( (stewardName, stewardsSeed), tdirWithPoolTxns) looper.add(stewardClient) looper.run(stewardClient.ensureConnectedToNodes()) return stewardClient, stewardWallet
def test_client_can_send_request_until_catchup(looper, poolTxnClientData, tdirWithPoolTxns, txnPoolNodeSet): slow_catch_up(txnPoolNodeSet, 60) client, _ = new_client(poolTxnClientData, tdirWithPoolTxns) looper.add(client) req = random_requests(1)[0] with pytest.raises(TestException): looper.run(eventually(can_send_request, client, req, timeout=4)) cancel_slow_catch_up(txnPoolNodeSet) looper.run(eventually(can_send_request, client, req))
def test_node_load_consistent_time(tconf, change_checkpoint_freq, disable_node_monitor_config, looper, txnPoolNodeSet, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): # One of the reason memory grows is because spylog grows client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) client_batches = 300 txns_per_batch = 25 time_log = [] warm_up_batches = 10 tolerance_factor = 2 from pympler import asizeof for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) t = perf_counter() - s with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'.format( i + 1, txns_per_batch, t)) print('--------Memory Usage details start') for node in txnPoolNodeSet: # print(sys.getsizeof(node)) print('---Node {}-----'.format(node)) # print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1))) print(get_memory_usage(node, True, get_only_non_empty=True)) for r in node.replicas: print('---Replica {}-----'.format(r)) print(get_memory_usage(r, True, get_only_non_empty=True)) print('--------Memory Usage details end') if len(time_log) >= warm_up_batches: m = mean(time_log) sd = tolerance_factor * pstdev(time_log) assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd) time_log.append(t) # Since client checks inbox for sufficient replies, clear inbox so that # it takes constant time to check replies for each batch client.inBox.clear() client.txnLog.reset()
def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) nodes = txnPoolNodeSet x = nodes[-1] with capsys.disabled(): print("Stopping node {} with pool ledger size {}".format( x, x.poolManager.txnSeqNo)) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, x) looper.removeProdable(x) client_batches = 80 txns_per_batch = 10 for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'.format( i + 1, txns_per_batch, perf_counter() - s)) nodeHa, nodeCHa = HA(*x.nodestack.ha), HA(*x.clientstack.ha) newNode = TestNode(x.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(newNode) txnPoolNodeSet[-1] = newNode looper.run(checkNodesConnected(txnPoolNodeSet))
def testZStackNodeReconnection(tconf, looper, txnPoolNodeSet, client1, wallet1, tdir, client1Connected): sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1) npr = [n for n in txnPoolNodeSet if not n.hasPrimary] nodeToCrash = npr[0] idxToCrash = txnPoolNodeSet.index(nodeToCrash) otherNodes = [_ for _ in txnPoolNodeSet if _ != nodeToCrash] def checkFlakyConnected(conn=True): for node in otherNodes: if conn: assert nodeToCrash.nodestack.name in node.nodestack.connecteds else: assert nodeToCrash.nodestack.name not in node.nodestack.connecteds checkFlakyConnected(True) nodeToCrash.stop() logger.debug('Stopped node {}'.format(nodeToCrash)) looper.removeProdable(nodeToCrash) looper.runFor(1) stopNodes([nodeToCrash], looper) # TODO Select or create the timeout from 'waits'. Don't use constant. looper.run(eventually(checkFlakyConnected, False, retryWait=1, timeout=60)) looper.runFor(1) config_helper = PNodeConfigHelper(nodeToCrash.name, tconf, chroot=tdir) node = TestNode(nodeToCrash.name, ledger_dir=config_helper.ledger_dir, keys_dir=config_helper.keys_dir, genesis_dir=config_helper.genesis_dir, plugins_dir=config_helper.plugins_dir, config=tconf, ha=nodeToCrash.nodestack.ha, cliha=nodeToCrash.clientstack.ha) looper.add(node) txnPoolNodeSet[idxToCrash] = node # TODO Select or create the timeout from 'waits'. Don't use constant. looper.run(eventually(checkFlakyConnected, True, retryWait=2, timeout=50)) ensureElectionsDone(looper, txnPoolNodeSet, retryWait=2) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 10)
def test_node_load(looper, txnPoolNodeSet, tconf, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) client_batches = 150 txns_per_batch = 25 for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, perf_counter() - s))
def test_node_catchup_causes_no_desync(looper, txnPoolNodeSet, client1, wallet1, client1Connected, monkeypatch): """ Checks that transactions received by catchup do not break performance monitoring """ client, wallet = client1, wallet1 lagging_node = get_any_non_primary_node(txnPoolNodeSet) rest_nodes = set(txnPoolNodeSet).difference({lagging_node}) # Make master replica lagging by hiding all messages sent to it make_master_replica_lag(lagging_node) monkeypatch.setattr(lagging_node.master_replica, '_request_missing_three_phase_messages', lambda *x, **y: None) # Send some requests and check that all replicas except master executed it sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) waitNodeDataInequality(looper, lagging_node, *rest_nodes) looper.run(eventually(backup_replicas_run_forward, lagging_node)) # Disconnect lagging node, send some more requests and start it back # After start it should fall in a such state that it needs to make catchup disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, lagging_node, stopNode=False) looper.removeProdable(lagging_node) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) looper.add(lagging_node) reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, lagging_node) # Check that catchup done waitNodeDataEquality(looper, lagging_node, *rest_nodes) # Send some more requests to ensure that backup and master replicas # are in the same state sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) looper.run(eventually(replicas_synced, lagging_node)) # Check that master is not considered to be degraded assert not lagging_node.monitor.isMasterDegraded()
def restart_nodes(looper, nodeSet, restart_set, tconf, tdir, allPluginsPath, after_restart_timeout=None, per_add_timeout=None): for node_to_stop in restart_set: node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) rest_nodes = [n for n in nodeSet if n not in restart_set] for node_to_stop in restart_set: ensure_node_disconnected(looper, node_to_stop, nodeSet, timeout=2) if after_restart_timeout: looper.runFor(after_restart_timeout) for node_to_restart in restart_set: config_helper = PNodeConfigHelper(node_to_restart.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_restart.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_restart.nodestack.ha, cliha=node_to_restart.clientstack.ha) looper.add(restarted_node) idx = nodeSet.index(node_to_restart) nodeSet[idx] = restarted_node if per_add_timeout: looper.run( checkNodesConnected(rest_nodes + [restarted_node], customTimeout=per_add_timeout)) rest_nodes += [restarted_node] if not per_add_timeout: looper.run( checkNodesConnected(nodeSet, customTimeout=after_restart_timeout))
def test_node_load_after_one_node_drops_all_msgs( looper, txnPoolNodeSet, tconf, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) nodes = txnPoolNodeSet x = nodes[-1] with capsys.disabled(): print("Patching node {}".format(x)) def handleOneNodeMsg(self, wrappedMsg): # do nothing with an incoming node message pass x.handleOneNodeMsg = MethodType(handleOneNodeMsg, x) client_batches = 120 txns_per_batch = 25 for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, perf_counter() - s))
def test_node_requests_missing_three_phase_messages_after_long_disconnection( looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf, tdirWithPoolTxns, allPluginsPath): """ 2 of 4 nodes go down, so pool can not process any more incoming requests. A new request comes in. Test than waits for some time to ensure that PrePrepare was created long enough seconds to be dropped by time checker. Two stopped nodes come back alive. Another request comes in. Check that previously disconnected two nodes request missing PREPARES and PREPREPARES and the pool successfully handles both transactions. """ INIT_REQS_CNT = 10 MISSING_REQS_CNT = 1 REQS_AFTER_RECONNECT_CNT = 1 alive_nodes = [] disconnected_nodes = [] for node in txnPoolNodeSet: if node.hasPrimary is not None: alive_nodes.append(node) else: disconnected_nodes.append(node) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, INIT_REQS_CNT) waitNodeDataEquality(looper, disconnected_nodes[0], *txnPoolNodeSet) init_ledger_size = txnPoolNodeSet[0].domainLedger.size for node in disconnected_nodes: disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node, stopNode=False) looper.removeProdable(node) sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, MISSING_REQS_CNT) def check_pp_out_of_sync(alive_nodes, disconnected_nodes): def get_last_pp(node): return node.replicas._master_replica.lastPrePrepare last_3pc_key_alive = get_last_pp(alive_nodes[0]) for node in alive_nodes[1:]: assert get_last_pp(node) == last_3pc_key_alive last_3pc_key_diconnected = get_last_pp(disconnected_nodes[0]) assert last_3pc_key_diconnected != last_3pc_key_alive for node in disconnected_nodes[1:]: assert get_last_pp(node) == last_3pc_key_diconnected looper.run( eventually(check_pp_out_of_sync, alive_nodes, disconnected_nodes, retryWait=1, timeout=expectedPoolGetReadyTimeout(len(txnPoolNodeSet)))) preprepare_deviation = 4 tconf.ACCEPTABLE_DEVIATION_PREPREPARE_SECS = preprepare_deviation time.sleep(preprepare_deviation * 2) for node in disconnected_nodes: looper.add(node) for node in disconnected_nodes: reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, node) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, REQS_AFTER_RECONNECT_CNT) waitNodeDataEquality(looper, disconnected_nodes[0], *txnPoolNodeSet) for node in txnPoolNodeSet: assert node.domainLedger.size == (init_ledger_size + MISSING_REQS_CNT + REQS_AFTER_RECONNECT_CNT)
def test_client_can_send_request_no_catchup(looper, poolTxnClientData, tdirWithPoolTxns, txnPoolNodeSet): client, _ = new_client(poolTxnClientData, tdirWithPoolTxns) looper.add(client) req = random_requests(1)[0] looper.run(eventually(can_send_request, client, req))
def test_client_can_send_read_requests_no_catchup(looper, poolTxnClientData, tdirWithPoolTxns, txnPoolNodeSet): client, _ = new_client(poolTxnClientData, tdirWithPoolTxns) looper.add(client) looper.run(eventually(can_send_read_requests, client))
def test_node_load_consistent_time(tconf, change_checkpoint_freq, disable_node_monitor_config, looper, txnPoolNodeSet, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): # One of the reason memory grows is because spylog grows client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) looper.add(client) looper.run(client.ensureConnectedToNodes()) client_batches = 300 txns_per_batch = 25 time_log = [] warm_up_batches = 10 tolerance_factor = 2 print_detailed_memory_usage = False from pympler import tracker tr = tracker.SummaryTracker() node_methods_to_capture = [TestNode.executeBatch, TestNode.recordAndPropagate, TestNode.domainDynamicValidation, TestNode.domainRequestApplication] times = {n.name: {meth.__name__: [] for meth in node_methods_to_capture} for n in txnPoolNodeSet} for node in txnPoolNodeSet: for meth in node_methods_to_capture: meth_name = meth.__name__ patched = timeit(getattr(node, meth_name), times[node.name][meth_name]) setattr(node, meth_name, patched) for i in range(client_batches): s = perf_counter() sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txns_per_batch, override_timeout_limit=True) t = perf_counter() - s with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, t)) print('--------Memory Usage details start') for node in txnPoolNodeSet: # print(sys.getsizeof(node)) print('---Node {}-----'.format(node)) # print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1))) print( get_memory_usage( node, print_detailed_memory_usage, get_only_non_empty=True)) for r in node.replicas: print('---Replica {}-----'.format(r)) print( get_memory_usage( r, print_detailed_memory_usage, get_only_non_empty=True)) # if i % 3 == 0: # tr.print_diff() print('--------Memory Usage details end') for node in txnPoolNodeSet: for meth in node_methods_to_capture: ts = times[node.name][meth.__name__] print('{} {} {} {}'.format( node, meth.__name__, mean(ts), ts)) if len(time_log) >= warm_up_batches: m = mean(time_log) sd = tolerance_factor * pstdev(time_log) assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd) time_log.append(t) # Since client checks inbox for sufficient replies, clear inbox so that # it takes constant time to check replies for each batch client.inBox.clear() client.txnLog.reset()