def test_node_load_after_one_node_drops_all_msgs(looper, txnPoolNodeSet, tconf,
tdirWithPoolTxns,
allPluginsPath,
poolTxnStewardData, capsys):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
nodes = txnPoolNodeSet
x = nodes[-1]
with capsys.disabled():
print("Patching node {}".format(x))
def handleOneNodeMsg(self, wrappedMsg):
# do nothing with an incoming node message
pass
x.handleOneNodeMsg = MethodType(handleOneNodeMsg, x)
client_batches = 120
txns_per_batch = 25
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
txns_per_batch,
override_timeout_limit=True)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.format(
i + 1, txns_per_batch,
perf_counter() - s))
def nodeCreatedAfterSomeTxns(txnPoolNodeSet, tdirWithPoolTxns,
poolTxnStewardData, tconf, allPluginsPath,
request):
with Looper(debug=True) as looper:
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
txnCount = getValueFromModule(request, "txnCount", 5)
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
txnCount,
timeoutPerReq=25)
newStewardName = randomString()
newNodeName = "Epsilon"
newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
looper,
client,
wallet,
newStewardName,
newNodeName,
tdirWithPoolTxns,
tconf,
allPluginsPath=allPluginsPath,
autoStart=True)
yield looper, newNode, client, wallet, newStewardClient, \
newStewardWallet
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet,
tdirWithPoolTxns, poolTxnClientData):
"""
Client should not be able to connect to nodes even after it has changed
its HA. Since running on a local environment, only checking change of port.
Dont know how to change IP.
:return:
"""
# TODO: Check for change of IP too
# name, seed = poolTxnClientData
# signer = SimpleSigner(seed=seed)
# name = "testClient96541"
# ha = genHa()
# client = genTestClient(txnPoolNodeSet, signer=signer, ha=ha,
# tmpdir=tdirWithPoolTxns, usePoolLedger=True,
# name=name)
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithPoolTxns)
looper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
*txnPoolNodeSet)
basedirpath = client.basedirpath
looper.removeProdable(client)
# Removing RAET keep directory otherwise the client will use the same port
# since it will a directory of its name in the keep
shutil.rmtree(os.path.join(basedirpath, client.name), ignore_errors=True)
ha = genHa()
client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId,
ha=ha, tmpdir=tdirWithPoolTxns,
usePoolLedger=True, name=client.name)
looper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
*txnPoolNodeSet)
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet,
tdirWithPoolTxns, poolTxnClientData):
"""
Client should not be able to connect to nodes even after it has changed
its HA. Since running on a local environment, only checking change of port.
Dont know how to change IP.
:return:
"""
# TODO: Check for change of IP too
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithPoolTxns)
looper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
*txnPoolNodeSet)
basedirpath = client.basedirpath
looper.removeProdable(client)
# Removing RAET keep directory otherwise the client will use the same port
# since it will a directory of its name in the keep
shutil.rmtree(os.path.join(basedirpath, client.name), ignore_errors=True)
ha = genHa()
client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId,
ha=ha, tmpdir=tdirWithPoolTxns,
usePoolLedger=True, name=client.name)
looper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
*txnPoolNodeSet)
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames,
tdirWithPoolTxns, poolTxnData, noRetryReq):
"""
Client should not be using node registry but pool transaction file
:return:
"""
clients = []
for name in poolTxnClientNames:
seed = poolTxnData["seeds"][name].encode()
client, wallet = buildPoolClientAndWallet((name, seed),
tdirWithPoolTxns)
txnPoolNodesLooper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client,
*txnPoolNodeSet)
clients.append((client, wallet))
n = 250
timeOutPerReq = 3
for (client, wallet) in clients:
logger.debug("{} sending {} requests".format(client, n))
sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client,
n, 1, timeOutPerReq)
logger.debug("{} sent {} requests".format(client, n))
for node in txnPoolNodeSet:
logger.debug("{} has requests {} with size {}".
format(node, len(node.requests), get_size(node.requests)))
for replica in node.replicas:
logger.debug("{} has prepares {} with size {}".
format(replica, len(replica.prepares),
get_size(replica.prepares)))
logger.debug("{} has commits {} with size {}".
format(replica, len(replica.commits),
get_size(replica.commits)))
def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet,
tdirWithPoolTxns,
tdirWithClientPoolTxns,
poolTxnClientData):
"""
Client should not be able to connect to nodes even after it has changed
its HA. Since running on a local environment, only checking change of port.
Dont know how to change IP.
:return:
"""
# TODO: Check for change of IP too
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithClientPoolTxns)
looper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
*txnPoolNodeSet)
keys_dir = os.path.join(client.keys_dir, client.name)
client.stop()
looper.removeProdable(client)
ha = genHa()
client, _ = genTestClient(txnPoolNodeSet,
identifier=wallet.defaultId,
ha=ha,
tmpdir=tdirWithClientPoolTxns,
usePoolLedger=True,
name=client.name)
looper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
*txnPoolNodeSet)
def nodeCreatedAfterSomeTxns(looper, testNodeClass, do_post_node_creation,
txnPoolNodeSet, tdir, tdirWithClientPoolTxns,
poolTxnStewardData, tconf, allPluginsPath,
request):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithClientPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
txnCount = getValueFromModule(request, "txnCount", 5)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txnCount)
newStewardName = randomString()
newNodeName = "Epsilon"
newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
looper,
client,
wallet,
newStewardName,
newNodeName,
tdir,
tdirWithClientPoolTxns,
tconf,
nodeClass=testNodeClass,
allPluginsPath=allPluginsPath,
autoStart=True,
do_post_node_creation=do_post_node_creation)
yield looper, newNode, client, wallet, newStewardClient, \
newStewardWallet
def demote_primary_node(looper,
initial_pool_of_nodes,
pool_of_nodes,
poolTxnStewardNames,
poolTxnData,
tdirWithClientPoolTxns):
demoted_node = [node for node in pool_of_nodes if node.has_master_primary][0]
indx = initial_pool_of_nodes.index(demoted_node)
steward_name = poolTxnStewardNames[indx]
stewards_seed = poolTxnData["seeds"][steward_name].encode()
stewardClient, stewardWallet = buildPoolClientAndWallet(
(steward_name, stewards_seed), tdirWithClientPoolTxns)
looper.add(stewardClient)
looper.run(stewardClient.ensureConnectedToNodes())
node_data = {
ALIAS: demoted_node.name,
SERVICES: []
}
updateNodeData(looper, stewardClient,
stewardWallet, demoted_node, node_data)
pool_of_nodes = list(set(pool_of_nodes) - {demoted_node})
return pool_of_nodes
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames,
tdirWithPoolTxns, poolTxnData, noRetryReq):
"""
Client should not be using node registry but pool transaction file
:return:
"""
clients = []
for name in poolTxnClientNames:
seed = poolTxnData["seeds"][name].encode()
client, wallet = buildPoolClientAndWallet((name, seed),
tdirWithPoolTxns)
txnPoolNodesLooper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper,
client, *txnPoolNodeSet)
clients.append((client, wallet))
n = 250
timeOutPerReq = 3
for (client, wallet) in clients:
logger.debug("{} sending {} requests".format(client, n))
sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client,
n, 1, timeOutPerReq)
logger.debug("{} sent {} requests".format(client, n))
for node in txnPoolNodeSet:
logger.debug("{} has requests {} with size {}".format(
node, len(node.requests), get_size(node.requests)))
for replica in node.replicas:
logger.debug("{} has prepares {} with size {}".format(
replica, len(replica.prepares), get_size(replica.prepares)))
logger.debug("{} has commits {} with size {}".format(
replica, len(replica.commits), get_size(replica.commits)))
def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames,
tdirWithPoolTxns, poolTxnData, noRetryReq):
"""
Client should not be using node registry but pool transaction file
:return:
"""
clients = []
for name in poolTxnClientNames:
seed = poolTxnData["seeds"][name].encode()
client, wallet = buildPoolClientAndWallet((name, seed),
tdirWithPoolTxns)
txnPoolNodesLooper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper,
client, *txnPoolNodeSet)
clients.append((client, wallet))
numRequests = 250
fVal = 1
for (client, wallet) in clients:
logger.debug("{} sending {} requests".format(client, numRequests))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, numRequests)
logger.debug("{} sent {} requests".format(client, numRequests))
for node in txnPoolNodeSet:
logger.debug("{} has requests {} with size {}".format(
node, len(node.requests), get_size(node.requests)))
for replica in node.replicas.values():
logger.debug("{} has prepares {} with size {}".format(
replica, len(replica.prepares), get_size(replica.prepares)))
logger.debug("{} has commits {} with size {}".format(
replica, len(replica.commits), get_size(replica.commits)))
def clientAndWallet2(looper, poolTxnClientData, tdirWithClientPoolTxns):
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithClientPoolTxns)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
yield client, wallet
client.stop()
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet,
tdirWithPoolTxns, tconf, steward1,
stewardWallet, nodeThetaAdded,
poolTxnClientData):
newSteward, newStewardWallet, newNode = nodeThetaAdded
nodeNewHa, clientNewHa = genHa(2)
logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa,
clientNewHa))
# Making the change HA txn an confirming its succeeded
changeNodeHa(looper,
newSteward,
newStewardWallet,
newNode,
nodeHa=nodeNewHa,
clientHa=clientNewHa)
# Stopping existing nodes
for node in txnPoolNodeSet:
node.stop()
looper.removeProdable(node)
# Starting nodes again by creating `Node` objects since that simulates
# what happens when starting the node with script
restartedNodes = []
for node in txnPoolNodeSet[:-1]:
restartedNode = TestNode(node.name,
basedirpath=tdirWithPoolTxns,
config=tconf,
ha=node.nodestack.ha,
cliha=node.clientstack.ha)
looper.add(restartedNode)
restartedNodes.append(restartedNode)
# Starting the node whose HA was changed
node = TestNode(newNode.name,
basedirpath=tdirWithPoolTxns,
config=tconf,
ha=nodeNewHa,
cliha=clientNewHa)
looper.add(node)
restartedNodes.append(node)
looper.run(checkNodesConnected(restartedNodes))
looper.run(
eventually(checkNodeLedgersForEquality,
node,
*restartedNodes[:-1],
retryWait=1,
timeout=10))
# Building a new client that reads from the genesis txn file
# but is able to connect to all nodes
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithPoolTxns)
looper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
*restartedNodes)
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdir, tdirWithPoolTxns,
tdirWithClientPoolTxns, tconf, steward1,
stewardWallet, nodeThetaAdded,
poolTxnClientData):
newSteward, newStewardWallet, newNode = nodeThetaAdded
nodeNewHa, clientNewHa = genHa(2)
logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa,
clientNewHa))
# Making the change HA txn an confirming its succeeded
op = {
ALIAS: newNode.name,
NODE_IP: nodeNewHa.host,
NODE_PORT: nodeNewHa.port,
CLIENT_IP: clientNewHa.host,
CLIENT_PORT: clientNewHa.port,
}
updateNodeData(looper, newSteward, newStewardWallet, newNode,
op)
# Stopping existing nodes
for node in txnPoolNodeSet:
node.stop()
looper.removeProdable(node)
# Starting nodes again by creating `Node` objects since that simulates
# what happens when starting the node with script
restartedNodes = []
for node in txnPoolNodeSet[:-1]:
config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir)
restartedNode = TestNode(node.name,
config_helper=config_helper,
config=tconf, ha=node.nodestack.ha,
cliha=node.clientstack.ha)
looper.add(restartedNode)
restartedNodes.append(restartedNode)
# Starting the node whose HA was changed
config_helper = PNodeConfigHelper(newNode.name, tconf, chroot=tdir)
node = TestNode(newNode.name,
config_helper=config_helper,
config=tconf,
ha=nodeNewHa, cliha=clientNewHa)
looper.add(node)
restartedNodes.append(node)
looper.run(checkNodesConnected(restartedNodes))
waitNodeDataEquality(looper, node, *restartedNodes[:-1])
# Building a new client that reads from the genesis txn file
# but is able to connect to all nodes
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithClientPoolTxns)
looper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
*restartedNodes)
def stewardClientAndWallet(poolNodesCreated, looper, tdirWithDomainTxns,
poolTxnStewardData):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithDomainTxns,
clientClass=TestClient,
walletClass=Wallet)
client.registerObserver(wallet.handleIncomingReply)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
makePendingTxnsRequest(client, wallet)
return client, wallet
def poolAfterSomeTxns(looper, txnPoolNodesLooper, txnPoolNodeSet,
tdirWithClientPoolTxns, poolTxnStewardData,
allPluginsPath, request):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithClientPoolTxns,
clientClass=TestClient)
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.add(client)
looper.run(client.ensureConnectedToNodes())
txnCount = getValueFromModule(request, "txnCount", 5)
sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client,
txnCount)
yield looper, client, wallet
def stewardAndWalletForMasterNode(looper, poolTxnData, poolTxnStewardNames,
tdirWithClientPoolTxns, txnPoolNodeSet, txnPoolMasterNodes):
primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet)
master_node = txnPoolMasterNodes[0]
stewardName = poolTxnStewardNames[primariesIdxs[0]]
stewardsSeed = poolTxnData["seeds"][stewardName].encode()
stewardClient, stewardWallet = buildPoolClientAndWallet(
(stewardName, stewardsSeed), tdirWithClientPoolTxns)
looper.add(stewardClient)
looper.run(stewardClient.ensureConnectedToNodes())
return stewardClient, stewardWallet
def test_node_load_consistent_time(tconf, change_checkpoint_freq,
disable_node_monitor_config, looper,
txnPoolNodeSet, tdirWithPoolTxns,
allPluginsPath, poolTxnStewardData, capsys):
# One of the reason memory grows is because spylog grows
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
client_batches = 300
txns_per_batch = 25
time_log = []
warm_up_batches = 10
tolerance_factor = 2
from pympler import asizeof
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
txns_per_batch,
override_timeout_limit=True)
t = perf_counter() - s
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.format(
i + 1, txns_per_batch, t))
print('--------Memory Usage details start')
for node in txnPoolNodeSet:
# print(sys.getsizeof(node))
print('---Node {}-----'.format(node))
# print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1)))
print(get_memory_usage(node, True, get_only_non_empty=True))
for r in node.replicas:
print('---Replica {}-----'.format(r))
print(get_memory_usage(r, True, get_only_non_empty=True))
print('--------Memory Usage details end')
if len(time_log) >= warm_up_batches:
m = mean(time_log)
sd = tolerance_factor * pstdev(time_log)
assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd)
time_log.append(t)
# Since client checks inbox for sufficient replies, clear inbox so that
# it takes constant time to check replies for each batch
client.inBox.clear()
client.txnLog.reset()
def stewards_and_wallets(looper, txnPoolNodeSet, pool_txn_stewards_data,
tdirWithPoolTxns):
clients_and_wallets = []
for pool_txn_steward_data in pool_txn_stewards_data:
steward_client, steward_wallet = buildPoolClientAndWallet(
pool_txn_steward_data, tdirWithPoolTxns)
looper.add(steward_client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward_client,
*txnPoolNodeSet)
clients_and_wallets.append((steward_client, steward_wallet))
yield clients_and_wallets
for (client, wallet) in clients_and_wallets:
client.stop()
def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf,
tdirWithPoolTxns, allPluginsPath,
poolTxnStewardData, capsys):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
nodes = txnPoolNodeSet
x = nodes[-1]
with capsys.disabled():
print("Stopping node {} with pool ledger size {}".format(
x, x.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, x)
looper.removeProdable(x)
client_batches = 80
txns_per_batch = 10
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
txns_per_batch,
override_timeout_limit=True)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.format(
i + 1, txns_per_batch,
perf_counter() - s))
nodeHa, nodeCHa = HA(*x.nodestack.ha), HA(*x.clientstack.ha)
newNode = TestNode(x.name,
basedirpath=tdirWithPoolTxns,
base_data_dir=tdirWithPoolTxns,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(newNode)
txnPoolNodeSet[-1] = newNode
looper.run(checkNodesConnected(txnPoolNodeSet))
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet,
tdirWithPoolTxns, tconf, steward1,
stewardWallet, nodeThetaAdded,
poolTxnClientData):
newSteward, newStewardWallet, newNode = nodeThetaAdded
nodeNewHa, clientNewHa = genHa(2)
logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa,
clientNewHa))
# Making the change HA txn an confirming its succeeded
changeNodeHa(looper, newSteward, newStewardWallet, newNode,
nodeHa=nodeNewHa, clientHa=clientNewHa)
# Stopping existing nodes
for node in txnPoolNodeSet:
node.stop()
looper.removeProdable(node)
# Starting nodes again by creating `Node` objects since that simulates
# what happens when starting the node with script
restartedNodes = []
for node in txnPoolNodeSet[:-1]:
restartedNode = TestNode(node.name, basedirpath=tdirWithPoolTxns,
config=tconf, ha=node.nodestack.ha,
cliha=node.clientstack.ha)
looper.add(restartedNode)
restartedNodes.append(restartedNode)
# Starting the node whose HA was changed
node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf,
ha=nodeNewHa, cliha=clientNewHa)
looper.add(node)
restartedNodes.append(node)
looper.run(checkNodesConnected(restartedNodes))
looper.run(eventually(checkNodeLedgersForEquality, node,
*restartedNodes[:-1], retryWait=1, timeout=10))
# Building a new client that reads from the genesis txn file
# but is able to connect to all nodes
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithPoolTxns)
looper.add(client)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, client,
*restartedNodes)
def test_node_load(looper, txnPoolNodeSet, tconf,
tdirWithPoolTxns, allPluginsPath,
poolTxnStewardData, capsys):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
client_batches = 150
txns_per_batch = 25
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client,
txns_per_batch,
override_timeout_limit=True)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.
format(i + 1, txns_per_batch, perf_counter() - s))
def checkStewardAdded(looper, poolTxnStewardData, tdirWithPoolTxns):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
sigseed = b'55555555555555555555555555555555'
newSigner = SimpleSigner(sigseed)
op = {
TXN_TYPE: NYM,
ROLE: STEWARD,
TARGET_NYM: newSigner.verkey,
ALIAS: "Robert",
}
req = wallet.signOp(op)
client.submitReqs(req)
def chk():
assert client.getReply(*req.key) == (None, "NOT_FOUND")
looper.run(eventually(chk, retryWait=1, timeout=5))
def checkStewardAdded(poolTxnStewardData, tdirWithPoolTxns):
with Looper(debug=True) as looper:
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
sigseed = b'55555555555555555555555555555555'
newSigner = SimpleSigner(sigseed)
op = {
TXN_TYPE: NYM,
ROLE: STEWARD,
TARGET_NYM: newSigner.verkey,
ALIAS: "Robert",
}
req = wallet.signOp(op)
client.submitReqs(req)
def chk():
assert client.getReply(req.reqId) == (None, "NOT_FOUND")
looper.run(eventually(chk, retryWait=1, timeout=5))
def pre_check(tconf, looper, txnPoolNodeSet, tdirWithClientPoolTxns,
poolTxnStewardData):
# TODO: Maybe this needs to be extracted in another fixture
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithClientPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
for i in range(tconf.ProcessedBatchMapsToKeep - 1):
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1)
# All node maintain the same map from txn range to 3PC
looper.run(eventually(chk_if_equal_txn_to_3pc, txnPoolNodeSet))
for i in range(3):
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1)
# All node maintain the same map from txn range to 3PC and its equal to
# `tconf.ProcessedBatchMapsToKeep` even after sending more batches than
# `tconf.ProcessedBatchMapsToKeep`, which shows the garbage cleaning in
# action
looper.run(eventually(chk_if_equal_txn_to_3pc, txnPoolNodeSet,
tconf.ProcessedBatchMapsToKeep))
def nodeCreatedAfterSomeTxns(
txnPoolNodesLooper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnStewardData, tconf, allPluginsPath, request
):
# with Looper(debug=True) as looper:
client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient)
txnPoolNodesLooper.add(client)
txnPoolNodesLooper.run(client.ensureConnectedToNodes())
txnCount = getValueFromModule(request, "txnCount", 5)
sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client, txnCount, timeoutPerReq=25)
newStewardName = randomString()
newNodeName = "Epsilon"
newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
txnPoolNodesLooper,
client,
wallet,
newStewardName,
newNodeName,
tdirWithPoolTxns,
tconf,
allPluginsPath=allPluginsPath,
autoStart=True,
)
yield txnPoolNodesLooper, newNode, client, wallet, newStewardClient, newStewardWallet
def test_node_load_consistent_time(tconf, change_checkpoint_freq,
disable_node_monitor_config, looper,
txnPoolNodeSet, tdirWithPoolTxns,
allPluginsPath, poolTxnStewardData, capsys):
# One of the reason memory grows is because spylog grows
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
client_batches = 300
txns_per_batch = 25
time_log = []
warm_up_batches = 10
tolerance_factor = 2
print_detailed_memory_usage = False
from pympler import tracker
tr = tracker.SummaryTracker()
node_methods_to_capture = [
TestNode.executeBatch, TestNode.recordAndPropagate,
TestNode.domainDynamicValidation, TestNode.domainRequestApplication
]
times = {
n.name: {meth.__name__: []
for meth in node_methods_to_capture}
for n in txnPoolNodeSet
}
for node in txnPoolNodeSet:
for meth in node_methods_to_capture:
meth_name = meth.__name__
patched = timeit(getattr(node, meth_name),
times[node.name][meth_name])
setattr(node, meth_name, patched)
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
txns_per_batch,
override_timeout_limit=True)
t = perf_counter() - s
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.format(
i + 1, txns_per_batch, t))
print('--------Memory Usage details start')
for node in txnPoolNodeSet:
# print(sys.getsizeof(node))
print('---Node {}-----'.format(node))
# print('Requests {}'.format(asizeof.asizeof(node.requests, detail=1)))
print(
get_memory_usage(node,
print_detailed_memory_usage,
get_only_non_empty=True))
for r in node.replicas:
print('---Replica {}-----'.format(r))
print(
get_memory_usage(r,
print_detailed_memory_usage,
get_only_non_empty=True))
# if i % 3 == 0:
# tr.print_diff()
print('--------Memory Usage details end')
for node in txnPoolNodeSet:
for meth in node_methods_to_capture:
ts = times[node.name][meth.__name__]
print('{} {} {} {}'.format(node, meth.__name__, mean(ts),
ts))
if len(time_log) >= warm_up_batches:
m = mean(time_log)
sd = tolerance_factor * pstdev(time_log)
assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd)
time_log.append(t)
# Since client checks inbox for sufficient replies, clear inbox so that
# it takes constant time to check replies for each batch
client.inBox.clear()
client.txnLog.reset()
def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithPoolTxns):
return buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns)
def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData,
tdirWithPoolTxns):
return buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns)
def new_client(poolTxnClientData, tdirWithPoolTxns):
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithPoolTxns)
return (client, wallet)
def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithPoolTxns):
client, wallet = buildPoolClientAndWallet(poolTxnClientData,
tdirWithPoolTxns)
yield client, wallet
client.stop()
def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData,
tdirWithPoolTxns):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns)
yield client, wallet
client.stop()
def testStewardSuspendsNode(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf,
steward1, stewardWallet, nodeThetaAdded,
poolTxnStewardData, allPluginsPath):
newSteward, newStewardWallet, newNode = nodeThetaAdded
newNodeNym = hexToFriendly(newNode.nodestack.local.signer.verhex)
suspendNode(looper, newSteward, newStewardWallet, newNodeNym, newNode.name)
# Check suspended node does not exist in any nodeReg or remotes of
# nodes or clients
txnPoolNodeSet = txnPoolNodeSet[:-1]
for node in txnPoolNodeSet:
looper.run(eventually(checkNodeNotInNodeReg, node, newNode.name))
for client in (steward1, newSteward):
looper.run(eventually(checkNodeNotInNodeReg, client, newNode.name))
# Check a client can send request and receive replies
req = sendRandomRequest(newStewardWallet, newSteward)
checkSufficientRepliesForRequests(looper,
newSteward, [
req,
],
timeoutPerReq=10)
# Check that a restarted client or node does not connect to the suspended
# node
steward1.stop()
looper.removeProdable(steward1)
steward1, stewardWallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns)
looper.add(steward1)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
*txnPoolNodeSet)
looper.run(eventually(checkNodeNotInNodeReg, steward1, newNode.name))
newNode.stop()
looper.removeProdable(newNode)
# TODO: There is a bug that if a primary node is turned off, it sends
# duplicate Pre-Prepare and gets blacklisted. Here is the gist
# https://gist.github.com/lovesh/c16989616ebb6856f9fa2905c14dc4b7
oldNodeIdx, oldNode = [(i, n) for i, n in enumerate(txnPoolNodeSet)
if not n.hasPrimary][0]
oldNode.stop()
looper.removeProdable(oldNode)
oldNode = TestNode(oldNode.name,
basedirpath=tdirWithPoolTxns,
config=tconf,
pluginPaths=allPluginsPath)
looper.add(oldNode)
txnPoolNodeSet[oldNodeIdx] = oldNode
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.run(eventually(checkNodeNotInNodeReg, oldNode, newNode.name))
# Check that a node whose suspension is revoked can reconnect to other
# nodes and clients can also connect to that node
cancelNodeSuspension(looper, newSteward, newStewardWallet, newNodeNym,
newNode.name)
nodeTheta = TestNode(newNode.name,
basedirpath=tdirWithPoolTxns,
config=tconf,
pluginPaths=allPluginsPath,
ha=newNode.nodestack.ha,
cliha=newNode.clientstack.ha)
looper.add(nodeTheta)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=30))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
*txnPoolNodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward,
*txnPoolNodeSet)
def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithPoolTxns):
return buildPoolClientAndWallet(poolTxnClientData, tdirWithPoolTxns)
def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData,
tdirWithPoolTxns):
return buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns)