def test_state_regenerated_from_ledger(looper, txnPoolNodeSet, client1,
wallet1, client1Connected, tconf,
tdirWithPoolTxns, allPluginsPath):
"""
Node loses its state database but recreates it from ledger after start
"""
sent_batches = 10
send_reqs_batches_and_get_suff_replies(looper, wallet1, client1,
5 * sent_batches, sent_batches)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
node_to_stop = txnPoolNodeSet[-1]
node_state = node_to_stop.states[DOMAIN_LEDGER_ID]
assert not node_state.isEmpty
state_db_path = node_state._kv.db_path
nodeHa, nodeCHa = HA(*node_to_stop.nodestack.ha), HA(
*node_to_stop.clientstack.ha)
node_to_stop.stop()
looper.removeProdable(node_to_stop)
shutil.rmtree(state_db_path)
restarted_node = TestNode(node_to_stop.name,
basedirpath=tdirWithPoolTxns,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(restarted_node)
txnPoolNodeSet[-1] = restarted_node
looper.run(checkNodesConnected(txnPoolNodeSet))
waitNodeDataEquality(looper, restarted_node, *txnPoolNodeSet[:-1])
def testZStackNodeReconnection(tconf, looper, txnPoolNodeSet, client1, wallet1,
tdirWithPoolTxns, client1Connected):
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1)
npr = [n for n in txnPoolNodeSet if not n.hasPrimary]
nodeToCrash = npr[0]
idxToCrash = txnPoolNodeSet.index(nodeToCrash)
otherNodes = [_ for _ in txnPoolNodeSet if _ != nodeToCrash]
def checkFlakyConnected(conn=True):
for node in otherNodes:
if conn:
assert nodeToCrash.nodestack.name in node.nodestack.connecteds
else:
assert nodeToCrash.nodestack.name not in node.nodestack.connecteds
checkFlakyConnected(True)
nodeToCrash.stop()
looper.removeProdable(nodeToCrash)
looper.runFor(1)
looper.run(eventually(checkFlakyConnected, False, retryWait=1, timeout=35))
looper.runFor(1)
node = TestNode(nodeToCrash.name,
basedirpath=tdirWithPoolTxns,
config=tconf,
ha=nodeToCrash.nodestack.ha,
cliha=nodeToCrash.clientstack.ha)
looper.add(node)
txnPoolNodeSet[idxToCrash] = node
looper.run(eventually(checkFlakyConnected, True, retryWait=2, timeout=50))
ensureElectionsDone(looper, txnPoolNodeSet, retryWait=2, timeout=50)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1)
checkNodesSendingCommits(txnPoolNodeSet)
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdir, tconf,
sdk_pool_handle, sdk_wallet_client,
sdk_wallet_steward):
new_steward_wallet, new_node = \
sdk_add_new_steward_and_node(looper,
sdk_pool_handle,
sdk_wallet_steward,
'AnotherSteward' + randomString(4),
'AnotherNode' + randomString(4),
tdir,
tconf)
txnPoolNodeSet.append(new_node)
looper.run(checkNodesConnected(txnPoolNodeSet))
sdk_pool_refresh(looper, sdk_pool_handle)
node_new_ha, client_new_ha = genHa(2)
logger.debug("{} changing HAs to {} {}".format(new_node, node_new_ha,
client_new_ha))
# Making the change HA txn an confirming its succeeded
node_dest = hexToFriendly(new_node.nodestack.verhex)
sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle,
node_dest, new_node.name, node_new_ha.host,
node_new_ha.port, client_new_ha.host,
client_new_ha.port)
# Stopping existing nodes
for node in txnPoolNodeSet:
node.stop()
looper.removeProdable(node)
# Starting nodes again by creating `Node` objects since that simulates
# what happens when starting the node with script
restartedNodes = []
for node in txnPoolNodeSet[:-1]:
config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir)
restartedNode = TestNode(node.name,
config_helper=config_helper,
config=tconf,
ha=node.nodestack.ha,
cliha=node.clientstack.ha)
looper.add(restartedNode)
restartedNodes.append(restartedNode)
# Starting the node whose HA was changed
config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir)
node = TestNode(new_node.name,
config_helper=config_helper,
config=tconf,
ha=node_new_ha,
cliha=client_new_ha)
looper.add(node)
restartedNodes.append(node)
looper.run(checkNodesConnected(restartedNodes))
waitNodeDataEquality(looper, node, *restartedNodes[:-1])
sdk_pool_refresh(looper, sdk_pool_handle)
sdk_ensure_pool_functional(looper, restartedNodes, sdk_wallet_client,
sdk_pool_handle)
def clientAndWallet2(looper, poolTxnClientData, tdirWithClientPoolTxns):
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithClientPoolTxns)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
yield client, wallet
client.stop()
def test_node_load_after_add_then_disconnect(newNodeCaughtUp, txnPoolNodeSet,
tconf, looper, client1, wallet1,
client1Connected,
tdirWithPoolTxns, allPluginsPath,
poolTxnStewardData, capsys):
"""
A node that restarts after some transactions should eventually get the
transactions which happened while it was down
:return:
"""
new_node = newNodeCaughtUp
with capsys.disabled():
print("Stopping node {} with pool ledger size {}".
format(new_node, new_node.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node)
looper.removeProdable(new_node)
client_batches = 80
txns_per_batch = 10
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1,
txns_per_batch,
override_timeout_limit=True)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.
format(i + 1, txns_per_batch, perf_counter() - s))
with capsys.disabled():
print("Starting the stopped node, {}".format(new_node))
nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha)
new_node = TestNode(
new_node.name,
basedirpath=tdirWithPoolTxns,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(new_node)
txnPoolNodeSet[-1] = new_node
# Delay catchup reply processing so LedgerState does not change
delay_catchup_reply = 5
new_node.nodeIbStasher.delay(cr_delay(delay_catchup_reply))
looper.run(checkNodesConnected(txnPoolNodeSet))
# Make sure ledger starts syncing (sufficient consistency proofs received)
looper.run(eventually(check_ledger_state, new_node, DOMAIN_LEDGER_ID,
LedgerState.syncing, retryWait=.5, timeout=5))
# Not accurate timeout but a conservative one
timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \
2 * delay_catchup_reply
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4],
customTimeout=timeout)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5)
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4])
def test_primary_selection_after_primary_demotion_and_pool_restart(
looper, txnPoolNodeSet, stewardAndWalletForMasterNode,
txnPoolMasterNodes, tdir, tconf):
"""
Demote primary and restart the pool.
Pool should select new primary and have viewNo=0 after restart.
"""
logger.info(
"1. turn off the node which has primary replica for master instanse")
master_node = txnPoolMasterNodes[0]
client, wallet = stewardAndWalletForMasterNode
node_data = {ALIAS: master_node.name, SERVICES: []}
updateNodeData(looper, client, wallet, master_node, node_data)
restNodes = [
node for node in txnPoolNodeSet if node.name != master_node.name
]
ensureElectionsDone(looper, restNodes)
# ensure pool is working properly
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3)
logger.info("2. restart pool")
# Stopping existing nodes
for node in txnPoolNodeSet:
node.stop()
looper.removeProdable(node)
# Starting nodes again by creating `Node` objects since that simulates
# what happens when starting the node with script
restartedNodes = []
for node in txnPoolNodeSet:
config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir)
restartedNode = TestNode(node.name,
config_helper=config_helper,
config=tconf,
ha=node.nodestack.ha,
cliha=node.clientstack.ha)
looper.add(restartedNode)
restartedNodes.append(restartedNode)
restNodes = [
node for node in restartedNodes if node.name != master_node.name
]
looper.run(checkNodesConnected(restNodes))
ensureElectionsDone(looper, restNodes)
checkViewNoForNodes(restNodes, 0)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3)
primariesIdxs = getPrimaryNodesIdxs(restNodes)
assert restNodes[primariesIdxs[0]].name != master_node.name
def test_client_can_not_send_write_requests_until_catchup(
looper, poolTxnClientData, tdirWithPoolTxns, txnPoolNodeSet):
slow_catch_up(txnPoolNodeSet, 60)
client, _ = new_client(poolTxnClientData, tdirWithPoolTxns)
looper.add(client)
with pytest.raises(TestException):
looper.run(eventually(can_send_write_requests, client, timeout=4))
cancel_slow_catch_up(txnPoolNodeSet)
looper.run(eventually(can_send_write_requests, client))
def stewardAndWalletForMasterNode(looper, poolTxnData, poolTxnStewardNames,
tdirWithPoolTxns, txnPoolNodeSet,
txnPoolMasterNodes):
primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet)
master_node = txnPoolMasterNodes[0]
stewardName = poolTxnStewardNames[primariesIdxs[0]]
stewardsSeed = poolTxnData["seeds"][stewardName].encode()
stewardClient, stewardWallet = buildPoolClientAndWallet(
(stewardName, stewardsSeed), tdirWithPoolTxns)
looper.add(stewardClient)
looper.run(stewardClient.ensureConnectedToNodes())
return stewardClient, stewardWallet
def test_client_can_send_request_until_catchup(looper, poolTxnClientData,
tdirWithPoolTxns,
txnPoolNodeSet):
slow_catch_up(txnPoolNodeSet, 60)
client, _ = new_client(poolTxnClientData, tdirWithPoolTxns)
looper.add(client)
req = random_requests(1)[0]
with pytest.raises(TestException):
looper.run(eventually(can_send_request, client, req, timeout=4))
cancel_slow_catch_up(txnPoolNodeSet)
looper.run(eventually(can_send_request, client, req))
def test_node_load_consistent_time(tconf, change_checkpoint_freq,
disable_node_monitor_config, looper,
txnPoolNodeSet, tdirWithPoolTxns,
allPluginsPath, poolTxnStewardData, capsys):
# One of the reason memory grows is because spylog grows
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
client_batches = 300
txns_per_batch = 25
time_log = []
warm_up_batches = 10
tolerance_factor = 2
from pympler import asizeof
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
txns_per_batch,
override_timeout_limit=True)
t = perf_counter() - s
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.format(
i + 1, txns_per_batch, t))
print('--------Memory Usage details start')
for node in txnPoolNodeSet:
# print(sys.getsizeof(node))
print('---Node {}-----'.format(node))
# print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1)))
print(get_memory_usage(node, True, get_only_non_empty=True))
for r in node.replicas:
print('---Replica {}-----'.format(r))
print(get_memory_usage(r, True, get_only_non_empty=True))
print('--------Memory Usage details end')
if len(time_log) >= warm_up_batches:
m = mean(time_log)
sd = tolerance_factor * pstdev(time_log)
assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd)
time_log.append(t)
# Since client checks inbox for sufficient replies, clear inbox so that
# it takes constant time to check replies for each batch
client.inBox.clear()
client.txnLog.reset()
def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf,
tdirWithPoolTxns, allPluginsPath,
poolTxnStewardData, capsys):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
nodes = txnPoolNodeSet
x = nodes[-1]
with capsys.disabled():
print("Stopping node {} with pool ledger size {}".format(
x, x.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, x)
looper.removeProdable(x)
client_batches = 80
txns_per_batch = 10
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
txns_per_batch,
override_timeout_limit=True)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.format(
i + 1, txns_per_batch,
perf_counter() - s))
nodeHa, nodeCHa = HA(*x.nodestack.ha), HA(*x.clientstack.ha)
newNode = TestNode(x.name,
basedirpath=tdirWithPoolTxns,
base_data_dir=tdirWithPoolTxns,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(newNode)
txnPoolNodeSet[-1] = newNode
looper.run(checkNodesConnected(txnPoolNodeSet))
def testZStackNodeReconnection(tconf, looper, txnPoolNodeSet, client1, wallet1,
tdir, client1Connected):
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1)
npr = [n for n in txnPoolNodeSet if not n.hasPrimary]
nodeToCrash = npr[0]
idxToCrash = txnPoolNodeSet.index(nodeToCrash)
otherNodes = [_ for _ in txnPoolNodeSet if _ != nodeToCrash]
def checkFlakyConnected(conn=True):
for node in otherNodes:
if conn:
assert nodeToCrash.nodestack.name in node.nodestack.connecteds
else:
assert nodeToCrash.nodestack.name not in node.nodestack.connecteds
checkFlakyConnected(True)
nodeToCrash.stop()
logger.debug('Stopped node {}'.format(nodeToCrash))
looper.removeProdable(nodeToCrash)
looper.runFor(1)
stopNodes([nodeToCrash], looper)
# TODO Select or create the timeout from 'waits'. Don't use constant.
looper.run(eventually(checkFlakyConnected, False, retryWait=1, timeout=60))
looper.runFor(1)
config_helper = PNodeConfigHelper(nodeToCrash.name, tconf, chroot=tdir)
node = TestNode(nodeToCrash.name,
ledger_dir=config_helper.ledger_dir,
keys_dir=config_helper.keys_dir,
genesis_dir=config_helper.genesis_dir,
plugins_dir=config_helper.plugins_dir,
config=tconf,
ha=nodeToCrash.nodestack.ha,
cliha=nodeToCrash.clientstack.ha)
looper.add(node)
txnPoolNodeSet[idxToCrash] = node
# TODO Select or create the timeout from 'waits'. Don't use constant.
looper.run(eventually(checkFlakyConnected, True, retryWait=2, timeout=50))
ensureElectionsDone(looper, txnPoolNodeSet, retryWait=2)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 10)
def test_node_load(looper, txnPoolNodeSet, tconf,
tdirWithPoolTxns, allPluginsPath,
poolTxnStewardData, capsys):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
client_batches = 150
txns_per_batch = 25
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client,
txns_per_batch,
override_timeout_limit=True)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.
format(i + 1, txns_per_batch, perf_counter() - s))
def test_node_catchup_causes_no_desync(looper, txnPoolNodeSet, client1,
wallet1, client1Connected, monkeypatch):
"""
Checks that transactions received by catchup do not
break performance monitoring
"""
client, wallet = client1, wallet1
lagging_node = get_any_non_primary_node(txnPoolNodeSet)
rest_nodes = set(txnPoolNodeSet).difference({lagging_node})
# Make master replica lagging by hiding all messages sent to it
make_master_replica_lag(lagging_node)
monkeypatch.setattr(lagging_node.master_replica,
'_request_missing_three_phase_messages',
lambda *x, **y: None)
# Send some requests and check that all replicas except master executed it
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5)
waitNodeDataInequality(looper, lagging_node, *rest_nodes)
looper.run(eventually(backup_replicas_run_forward, lagging_node))
# Disconnect lagging node, send some more requests and start it back
# After start it should fall in a such state that it needs to make catchup
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
lagging_node,
stopNode=False)
looper.removeProdable(lagging_node)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5)
looper.add(lagging_node)
reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, lagging_node)
# Check that catchup done
waitNodeDataEquality(looper, lagging_node, *rest_nodes)
# Send some more requests to ensure that backup and master replicas
# are in the same state
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5)
looper.run(eventually(replicas_synced, lagging_node))
# Check that master is not considered to be degraded
assert not lagging_node.monitor.isMasterDegraded()
def restart_nodes(looper,
nodeSet,
restart_set,
tconf,
tdir,
allPluginsPath,
after_restart_timeout=None,
per_add_timeout=None):
for node_to_stop in restart_set:
node_to_stop.cleanupOnStopping = True
node_to_stop.stop()
looper.removeProdable(node_to_stop)
rest_nodes = [n for n in nodeSet if n not in restart_set]
for node_to_stop in restart_set:
ensure_node_disconnected(looper, node_to_stop, nodeSet, timeout=2)
if after_restart_timeout:
looper.runFor(after_restart_timeout)
for node_to_restart in restart_set:
config_helper = PNodeConfigHelper(node_to_restart.name,
tconf,
chroot=tdir)
restarted_node = TestNode(node_to_restart.name,
config_helper=config_helper,
config=tconf,
pluginPaths=allPluginsPath,
ha=node_to_restart.nodestack.ha,
cliha=node_to_restart.clientstack.ha)
looper.add(restarted_node)
idx = nodeSet.index(node_to_restart)
nodeSet[idx] = restarted_node
if per_add_timeout:
looper.run(
checkNodesConnected(rest_nodes + [restarted_node],
customTimeout=per_add_timeout))
rest_nodes += [restarted_node]
if not per_add_timeout:
looper.run(
checkNodesConnected(nodeSet, customTimeout=after_restart_timeout))
def test_node_load_after_one_node_drops_all_msgs(
looper,
txnPoolNodeSet,
tconf,
tdirWithPoolTxns,
allPluginsPath,
poolTxnStewardData,
capsys):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
nodes = txnPoolNodeSet
x = nodes[-1]
with capsys.disabled():
print("Patching node {}".format(x))
def handleOneNodeMsg(self, wrappedMsg):
# do nothing with an incoming node message
pass
x.handleOneNodeMsg = MethodType(handleOneNodeMsg, x)
client_batches = 120
txns_per_batch = 25
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client,
txns_per_batch,
override_timeout_limit=True)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.
format(i + 1, txns_per_batch, perf_counter() - s))
def test_node_requests_missing_three_phase_messages_after_long_disconnection(
looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf,
tdirWithPoolTxns, allPluginsPath):
"""
2 of 4 nodes go down, so pool can not process any more incoming requests.
A new request comes in.
Test than waits for some time to ensure that PrePrepare was created
long enough seconds to be dropped by time checker.
Two stopped nodes come back alive.
Another request comes in.
Check that previously disconnected two nodes request missing PREPARES and
PREPREPARES and the pool successfully handles both transactions.
"""
INIT_REQS_CNT = 10
MISSING_REQS_CNT = 1
REQS_AFTER_RECONNECT_CNT = 1
alive_nodes = []
disconnected_nodes = []
for node in txnPoolNodeSet:
if node.hasPrimary is not None:
alive_nodes.append(node)
else:
disconnected_nodes.append(node)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, INIT_REQS_CNT)
waitNodeDataEquality(looper, disconnected_nodes[0], *txnPoolNodeSet)
init_ledger_size = txnPoolNodeSet[0].domainLedger.size
for node in disconnected_nodes:
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
node,
stopNode=False)
looper.removeProdable(node)
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
MISSING_REQS_CNT)
def check_pp_out_of_sync(alive_nodes, disconnected_nodes):
def get_last_pp(node):
return node.replicas._master_replica.lastPrePrepare
last_3pc_key_alive = get_last_pp(alive_nodes[0])
for node in alive_nodes[1:]:
assert get_last_pp(node) == last_3pc_key_alive
last_3pc_key_diconnected = get_last_pp(disconnected_nodes[0])
assert last_3pc_key_diconnected != last_3pc_key_alive
for node in disconnected_nodes[1:]:
assert get_last_pp(node) == last_3pc_key_diconnected
looper.run(
eventually(check_pp_out_of_sync,
alive_nodes,
disconnected_nodes,
retryWait=1,
timeout=expectedPoolGetReadyTimeout(len(txnPoolNodeSet))))
preprepare_deviation = 4
tconf.ACCEPTABLE_DEVIATION_PREPREPARE_SECS = preprepare_deviation
time.sleep(preprepare_deviation * 2)
for node in disconnected_nodes:
looper.add(node)
for node in disconnected_nodes:
reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, node)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, REQS_AFTER_RECONNECT_CNT)
waitNodeDataEquality(looper, disconnected_nodes[0], *txnPoolNodeSet)
for node in txnPoolNodeSet:
assert node.domainLedger.size == (init_ledger_size + MISSING_REQS_CNT +
REQS_AFTER_RECONNECT_CNT)
def test_client_can_send_request_no_catchup(looper, poolTxnClientData,
tdirWithPoolTxns, txnPoolNodeSet):
client, _ = new_client(poolTxnClientData, tdirWithPoolTxns)
looper.add(client)
req = random_requests(1)[0]
looper.run(eventually(can_send_request, client, req))
def test_client_can_send_read_requests_no_catchup(looper, poolTxnClientData,
tdirWithPoolTxns,
txnPoolNodeSet):
client, _ = new_client(poolTxnClientData, tdirWithPoolTxns)
looper.add(client)
looper.run(eventually(can_send_read_requests, client))
def test_node_load_consistent_time(tconf, change_checkpoint_freq,
disable_node_monitor_config, looper,
txnPoolNodeSet, tdirWithPoolTxns,
allPluginsPath, poolTxnStewardData, capsys):
# One of the reason memory grows is because spylog grows
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
client_batches = 300
txns_per_batch = 25
time_log = []
warm_up_batches = 10
tolerance_factor = 2
print_detailed_memory_usage = False
from pympler import tracker
tr = tracker.SummaryTracker()
node_methods_to_capture = [TestNode.executeBatch,
TestNode.recordAndPropagate,
TestNode.domainDynamicValidation,
TestNode.domainRequestApplication]
times = {n.name: {meth.__name__: [] for meth in node_methods_to_capture}
for n in txnPoolNodeSet}
for node in txnPoolNodeSet:
for meth in node_methods_to_capture:
meth_name = meth.__name__
patched = timeit(getattr(node, meth_name),
times[node.name][meth_name])
setattr(node, meth_name, patched)
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client,
txns_per_batch,
override_timeout_limit=True)
t = perf_counter() - s
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.
format(i + 1, txns_per_batch, t))
print('--------Memory Usage details start')
for node in txnPoolNodeSet:
# print(sys.getsizeof(node))
print('---Node {}-----'.format(node))
# print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1)))
print(
get_memory_usage(
node,
print_detailed_memory_usage,
get_only_non_empty=True))
for r in node.replicas:
print('---Replica {}-----'.format(r))
print(
get_memory_usage(
r,
print_detailed_memory_usage,
get_only_non_empty=True))
# if i % 3 == 0:
# tr.print_diff()
print('--------Memory Usage details end')
for node in txnPoolNodeSet:
for meth in node_methods_to_capture:
ts = times[node.name][meth.__name__]
print('{} {} {} {}'.format(
node, meth.__name__, mean(ts), ts))
if len(time_log) >= warm_up_batches:
m = mean(time_log)
sd = tolerance_factor * pstdev(time_log)
assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd)
time_log.append(t)
# Since client checks inbox for sufficient replies, clear inbox so that
# it takes constant time to check replies for each batch
client.inBox.clear()
client.txnLog.reset()
