def testCatchupDelayedNodes(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns,
txnPoolCliNodeReg, tdirWithPoolTxns, tconf,
allPluginsPath):
"""
Node sends catchup request to other nodes for only those sequence numbers
that other nodes have. Have pool of connected nodes with some transactions
made and then two more nodes say X and Y will join where Y node will start
its catchup process after some time. The node starting late, i.e. Y should
not receive any catchup requests
:return:
"""
looper, _, _, _, client, wallet = nodeSetWithNodeAddedAfterSomeTxns
stewardXName = "testClientStewardX"
nodeXName = "Zeta"
stewardYName = "testClientStewardY"
nodeYName = "Eta"
stewardZName = "testClientStewardZ"
nodeZName = "Theta"
delayX = 45
delayY = 2
stewardX, nodeX = addNewStewardAndNode(looper,
client,
stewardXName,
nodeXName,
tdirWithPoolTxns,
tconf,
allPluginsPath,
autoStart=False)
stewardY, nodeY = addNewStewardAndNode(looper,
client,
stewardYName,
nodeYName,
tdirWithPoolTxns,
tconf,
allPluginsPath,
autoStart=False)
nodeX.nodeIbStasher.delay(cpDelay(delayX))
nodeY.nodeIbStasher.delay(cpDelay(delayY))
looper.add(nodeX)
looper.add(nodeY)
txnPoolNodeSet.append(nodeX)
txnPoolNodeSet.append(nodeY)
timeout = waits.expectedPoolCatchupTime(
len(txnPoolNodeSet)) + delayX + delayY
looper.run(checkNodesConnected(txnPoolNodeSet, customTimeout=timeout))
logger.debug("Stopping 2 newest nodes, {} and {}".format(
nodeX.name, nodeY.name))
nodeX.stop()
nodeY.stop()
logger.debug("Sending requests")
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 50)
logger.debug("Starting the 2 stopped nodes, {} and {}".format(
nodeX.name, nodeY.name))
nodeX.start(looper.loop)
nodeY.start(looper.loop)
waitNodeDataEquality(looper, nodeX, *txnPoolNodeSet[:5])
waitNodeDataEquality(looper, nodeY, *txnPoolNodeSet[:5])
def testClientConnectsToNewNode(looper, txnPoolNodeSet, tdirWithPoolTxns,
tconf, steward1, stewardWallet,
allPluginsPath):
"""
A client should be able to connect to a newly added node
"""
newStewardName = "testClientSteward" + randomString(3)
newNodeName = "Epsilon"
oldNodeReg = copy(steward1.nodeReg)
newSteward, newStewardWallet, newNode = addNewStewardAndNode(
looper, steward1, stewardWallet, newStewardName, newNodeName,
tdirWithPoolTxns, tconf, allPluginsPath)
txnPoolNodeSet.append(newNode)
looper.run(checkNodesConnected(txnPoolNodeSet))
logger.debug("{} connected to the pool".format(newNode))
def chkNodeRegRecvd():
assert (len(steward1.nodeReg) - len(oldNodeReg)) == 1
assert (newNode.name + CLIENT_STACK_SUFFIX) in steward1.nodeReg
looper.run(eventually(chkNodeRegRecvd, retryWait=1, timeout=5))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
*txnPoolNodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward,
*txnPoolNodeSet)
def testAdd2NewNodes(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1,
stewardWallet, allPluginsPath):
"""
Add 2 new nodes to trigger replica addition and primary election
"""
for nodeName in ("Zeta", "Eta"):
newStewardName = "testClientSteward" + randomString(3)
newSteward, newStewardWallet, newNode = addNewStewardAndNode(
looper, steward1, stewardWallet, newStewardName, nodeName,
tdirWithPoolTxns, tconf, allPluginsPath)
txnPoolNodeSet.append(newNode)
looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=30))
logger.debug("{} connected to the pool".format(newNode))
looper.run(
eventually(checkNodeLedgersForEquality,
newNode,
*txnPoolNodeSet[:-1],
retryWait=1,
timeout=7))
f = getMaxFailures(len(txnPoolNodeSet))
def checkFValue():
for node in txnPoolNodeSet:
assert node.f == f
assert len(node.replicas) == (f + 1)
looper.run(eventually(checkFValue, retryWait=1, timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1, timeout=5)
def testAddInactiveNodeThenActivate(looper, txnPoolNodeSet, tdirWithPoolTxns,
tconf, steward1, stewardWallet,
allPluginsPath):
newStewardName = "testClientSteward" + randomString(3)
newNodeName = "Kappa"
# adding a new node without SERVICES field
# it means the node is in the inactive state
def del_services(op):
del op[DATA][SERVICES]
newSteward, newStewardWallet, newNode = \
addNewStewardAndNode(looper,
steward1, stewardWallet,
newStewardName, newNodeName,
tdirWithPoolTxns, tconf, allPluginsPath,
transformNodeOpFunc=del_services)
looper.run(checkNodesConnected(txnPoolNodeSet))
# turn the new node on
node_data = {ALIAS: newNode.name, SERVICES: [VALIDATOR]}
updateNodeDataAndReconnect(looper, newSteward, newStewardWallet, newNode,
node_data, tdirWithPoolTxns, tconf,
txnPoolNodeSet + [newNode])
def testClientConnectsToNewNode(looper, txnPoolNodeSet, tdirWithPoolTxns,
tconf, steward1, stewardWallet, allPluginsPath):
"""
A client should be able to connect to a newly added node
"""
newStewardName = "testClientSteward"+randomString(3)
newNodeName = "Epsilon"
oldNodeReg = copy(steward1.nodeReg)
newSteward, newStewardWallet, newNode = addNewStewardAndNode(looper,
steward1, stewardWallet,
newStewardName, newNodeName,
tdirWithPoolTxns, tconf,
allPluginsPath)
txnPoolNodeSet.append(newNode)
looper.run(checkNodesConnected(txnPoolNodeSet))
logger.debug("{} connected to the pool".format(newNode))
def chkNodeRegRecvd():
assert (len(steward1.nodeReg) - len(oldNodeReg)) == 1
assert (newNode.name + CLIENT_STACK_SUFFIX) in steward1.nodeReg
looper.run(eventually(chkNodeRegRecvd, retryWait=1, timeout=5))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
*txnPoolNodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward,
*txnPoolNodeSet)
def nodeCreatedAfterSomeTxns(txnPoolNodeSet, tdirWithPoolTxns,
poolTxnStewardData, tconf, allPluginsPath,
request):
with Looper(debug=True) as looper:
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
txnCount = getValueFromModule(request, "txnCount", 5)
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
txnCount,
timeoutPerReq=25)
newStewardName = randomString()
newNodeName = "Epsilon"
newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
looper,
client,
wallet,
newStewardName,
newNodeName,
tdirWithPoolTxns,
tconf,
allPluginsPath=allPluginsPath,
autoStart=True)
yield looper, newNode, client, wallet, newStewardClient, \
newStewardWallet
def nodeCreatedAfterSomeTxns(looper, testNodeClass, do_post_node_creation,
txnPoolNodeSet, tdir, tdirWithClientPoolTxns,
poolTxnStewardData, tconf, allPluginsPath,
request):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithClientPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
txnCount = getValueFromModule(request, "txnCount", 5)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, txnCount)
newStewardName = randomString()
newNodeName = "Epsilon"
newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
looper,
client,
wallet,
newStewardName,
newNodeName,
tdir,
tdirWithClientPoolTxns,
tconf,
nodeClass=testNodeClass,
allPluginsPath=allPluginsPath,
autoStart=True,
do_post_node_creation=do_post_node_creation)
yield looper, newNode, client, wallet, newStewardClient, \
newStewardWallet
def nodeThetaAdded(looper,
txnPoolNodeSet,
tdirWithPoolTxns,
tconf,
steward1,
stewardWallet,
allPluginsPath,
testNodeClass=None,
testClientClass=None,
name=None):
newStewardName = "testClientSteward" + randomString(3)
newNodeName = name or "Theta"
newSteward, newStewardWallet, newNode = addNewStewardAndNode(
looper,
steward1,
stewardWallet,
newStewardName,
newNodeName,
tdirWithPoolTxns,
tconf,
allPluginsPath,
nodeClass=testNodeClass,
clientClass=testClientClass)
txnPoolNodeSet.append(newNode)
looper.run(checkNodesConnected(txnPoolNodeSet))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
*txnPoolNodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward,
*txnPoolNodeSet)
return newSteward, newStewardWallet, newNode
def testAdd2NewNodes(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1,
stewardWallet, allPluginsPath):
"""
Add 2 new nodes to trigger replica addition and primary election
"""
for nodeName in ("Zeta", "Eta"):
newStewardName = "testClientSteward"+randomString(3)
newSteward, newStewardWallet, newNode = addNewStewardAndNode(looper,
steward1,
stewardWallet,
newStewardName,
nodeName,
tdirWithPoolTxns, tconf,
allPluginsPath)
txnPoolNodeSet.append(newNode)
looper.run(checkNodesConnected(txnPoolNodeSet))
logger.debug("{} connected to the pool".format(newNode))
looper.run(eventually(checkNodeLedgersForEquality, newNode,
*txnPoolNodeSet[:-1], retryWait=1, timeout=7))
f = getMaxFailures(len(txnPoolNodeSet))
def checkFValue():
for node in txnPoolNodeSet:
assert node.f == f
assert len(node.replicas) == (f + 1)
looper.run(eventually(checkFValue, retryWait=1, timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1,
timeout=5)
def test_add_node_to_pool_with_large_ppseqno_diff_views(
do_view_change, looper, txnPoolNodeSet, tconf, steward1, stewardWallet,
tdir, client_tdir, allPluginsPath):
"""
Adding a node to the pool while ppSeqNo is big caused a node to stash all the
requests because of incorrect watermarks limits set.
The case of view_no == 0 is special.
The test emulates big ppSeqNo number, adds a node and checks all the pool nodes
are functional. The test is run with several starting view_no, including 0
"""
# TODO: for now this test will use old client api, after moving node txn to sdk it will be rewritten
ensure_several_view_change(looper,
txnPoolNodeSet,
do_view_change,
custom_timeout=tconf.VIEW_CHANGE_TIMEOUT)
big_ppseqno = tconf.LOG_SIZE * 2 + 2345
cur_ppseqno = _get_ppseqno(txnPoolNodeSet)
assert (big_ppseqno > cur_ppseqno)
# ensure pool is working properly
sendReqsToNodesAndVerifySuffReplies(looper,
stewardWallet,
steward1,
numReqs=3)
assert (cur_ppseqno < _get_ppseqno(txnPoolNodeSet))
_set_ppseqno(txnPoolNodeSet, big_ppseqno)
cur_ppseqno = _get_ppseqno(txnPoolNodeSet)
assert (big_ppseqno == cur_ppseqno)
sendReqsToNodesAndVerifySuffReplies(looper,
stewardWallet,
steward1,
numReqs=3)
assert (cur_ppseqno < _get_ppseqno(txnPoolNodeSet))
new_steward_name = "testClientSteward" + randomString(4)
new_node_name = "TestTheta" + randomString(4)
new_steward, new_steward_wallet, new_node = \
addNewStewardAndNode(looper, steward1, stewardWallet, new_steward_name,
new_node_name, tdir, client_tdir, tconf, allPluginsPath)
txnPoolNodeSet.append(new_node)
looper.run(checkNodesConnected(txnPoolNodeSet))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
*txnPoolNodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, new_steward,
*txnPoolNodeSet)
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
sendReqsToNodesAndVerifySuffReplies(looper,
stewardWallet,
steward1,
numReqs=3)
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
def test_second_checkpoint_after_catchup_can_be_stabilized(
chkFreqPatched, looper, txnPoolNodeSet, steward1, stewardWallet,
client1, wallet1, client1Connected, tdir, client_tdir, tconf,
allPluginsPath):
_, _, epsilon = addNewStewardAndNode(looper, steward1, stewardWallet,
'EpsilonSteward', 'Epsilon', tdir,
client_tdir, tconf, allPluginsPath)
txnPoolNodeSet.append(epsilon)
looper.run(checkNodesConnected(txnPoolNodeSet))
waitNodeDataEquality(looper, epsilon, *txnPoolNodeSet[:-1])
# Epsilon did not participate in ordering of the batch with EpsilonSteward
# NYM transaction and the batch with Epsilon NODE transaction.
# Epsilon got these transactions via catch-up.
for replica in epsilon.replicas:
assert len(replica.checkpoints) == 0
assert replica.h == 2
assert replica.H == 17
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 7)
stabilization_timeout = \
waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.runFor(stabilization_timeout)
for replica in epsilon.replicas:
assert len(replica.checkpoints) == 2
keys_iter = iter(replica.checkpoints)
assert next(keys_iter) == (3, 5)
assert replica.checkpoints[3, 5].seqNo == 5
assert replica.checkpoints[3, 5].digest is None
assert replica.checkpoints[3, 5].isStable is False
assert next(keys_iter) == (6, 10)
assert replica.checkpoints[6, 10].seqNo == 9
assert replica.checkpoints[6, 10].digest is None
assert replica.checkpoints[6, 10].isStable is False
assert replica.h == 2
assert replica.H == 17
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1)
looper.runFor(stabilization_timeout)
for replica in epsilon.replicas:
assert len(replica.checkpoints) == 1
keys_iter = iter(replica.checkpoints)
assert next(keys_iter) == (6, 10)
assert replica.checkpoints[6, 10].seqNo == 10
assert replica.checkpoints[6, 10].digest is not None
assert replica.checkpoints[6, 10].isStable is True
assert replica.h == 10
assert replica.H == 25
def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet,
nodeSetWithNodeAddedAfterSomeTxns,
newNodeCaughtUp, tdirWithPoolTxns,
tconf, allPluginsPath):
"""
Node discards 3-phase and election messages from view nos that it does not
know of (view nos before it joined the pool)
:return:
"""
looper, nodeX, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns
viewNo = nodeX.viewNo
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, 10)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 4)
looper.run(eventually(partial(checkViewNoForNodes, txnPoolNodeSet,
viewNo + 1), retryWait=1, timeout=20))
newStewardName = "testClientSteward" + randomString(3)
nodeName = "Theta"
_, _, nodeTheta = addNewStewardAndNode(looper, client,
wallet,
newStewardName,
nodeName,
tdirWithPoolTxns, tconf,
allPluginsPath)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.run(client.ensureConnectedToNodes())
looper.run(eventually(checkNodeLedgersForEquality, nodeTheta,
*txnPoolNodeSet[:-1], retryWait=1, timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1,
timeout=10)
electMsg = Nomination(nodeX.name, 0, viewNo)
threePMsg = PrePrepare(
0,
viewNo,
10,
wallet.defaultId,
wallet._getIdData().lastReqId+1,
"random digest",
time.time()
)
ridTheta = nodeX.nodestack.getRemote(nodeTheta.name).uid
nodeX.send(electMsg, ridTheta)
nodeX.send(threePMsg, ridTheta)
nodeX.send(electMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], electMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
nodeX.send(threePMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], threePMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet,
nodeSetWithNodeAddedAfterSomeTxns,
newNodeCaughtUp, tdirWithPoolTxns,
tconf, allPluginsPath):
"""
Node discards 3-phase and election messages from view nos that it does not
know of (view nos before it joined the pool)
:return:
"""
looper, nodeX, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns
viewNo = nodeX.viewNo
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, 10)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 4)
looper.run(eventually(partial(checkViewNoForNodes, txnPoolNodeSet,
viewNo + 1), retryWait=1, timeout=20))
newStewardName = "testClientSteward" + randomString(3)
nodeName = "Theta"
_, _, nodeTheta = addNewStewardAndNode(looper, client,
wallet,
newStewardName,
nodeName,
tdirWithPoolTxns, tconf,
allPluginsPath)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.run(client.ensureConnectedToNodes())
looper.run(eventually(checkNodeLedgersForEquality, nodeTheta,
*txnPoolNodeSet[:-1], retryWait=1, timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1,
timeout=10)
electMsg = Nomination(nodeX.name, 0, viewNo)
threePMsg = PrePrepare(
0,
viewNo,
10,
wallet.defaultId,
wallet._getIdData().lastReqId+1,
"random digest",
time.time()
)
ridTheta = nodeX.nodestack.getRemote(nodeTheta.name).uid
nodeX.send(electMsg, ridTheta)
nodeX.send(threePMsg, ridTheta)
nodeX.send(electMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], electMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
nodeX.send(threePMsg, ridTheta)
looper.run(eventually(checkDiscardMsg, [nodeTheta, ], threePMsg,
'un-acceptable viewNo', retryWait=1, timeout=5))
def testNewNodeCatchupWhileIncomingRequests(looper, txnPoolNodeSet, tdir,
tdirWithClientPoolTxns, tconf,
steward1, stewardWallet,
allPluginsPath):
"""
A new node joins while transactions are happening, its catchup requests
include till where it has to catchup, which would be less than the other
node's ledger size. In the meantime, the new node will stash all requests
"""
sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 5)
def chkAfterCall(self, req, frm):
r = self.processCatchupReq(req, frm)
typ = getattr(req, f.LEDGER_ID.nm)
if typ == DOMAIN_LEDGER_ID:
ledger = self.getLedgerForMsg(req)
assert req.catchupTill <= ledger.size
return r
for node in txnPoolNodeSet:
node.nodeMsgRouter.routes[CatchupReq] = \
types.MethodType(chkAfterCall, node.ledgerManager)
node.nodeIbStasher.delay(cqDelay(3))
print('Sending 5 requests')
sendRandomRequests(stewardWallet, steward1, 5)
looper.runFor(1)
newStewardName = randomString()
newNodeName = "Epsilon"
newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
looper,
steward1,
stewardWallet,
newStewardName,
newNodeName,
tdir,
tdirWithClientPoolTxns,
tconf,
allPluginsPath=allPluginsPath,
autoStart=True)
txnPoolNodeSet.append(newNode)
looper.runFor(2)
sendRandomRequests(stewardWallet, steward1, 5)
# TODO select or create a timeout for this case in 'waits'
looper.run(
eventually(checkNodeDataForEquality,
newNode,
*txnPoolNodeSet[:-1],
retryWait=1,
timeout=80))
assert newNode.spylog.count(TestNode.processStashedOrderedReqs) > 0
def add_new_node(looper, pool_nodes, steward, steward_wallet,
tdir, client_tdir, tconf, all_plugins_path):
name = randomString(6)
node_name = "Node-" + name
new_steward_name = "Steward-" + name
_, _, new_node = addNewStewardAndNode(looper, steward, steward_wallet,
new_steward_name, node_name,
tdir, client_tdir, tconf,
all_plugins_path)
pool_nodes.append(new_node)
looper.run(checkNodesConnected(pool_nodes))
waitNodeDataEquality(looper, new_node, *pool_nodes[:-1])
# The new node did not participate in ordering of the batch with
# the new steward NYM transaction and the batch with the new NODE
# transaction. The new node got these transactions via catch-up.
return new_node
def add_new_node(looper, nodes, steward, steward_wallet,
tdir, client_tdir, tconf, all_plugins_path, name=None):
node_name = name or randomString(5)
new_steward_name = "testClientSteward" + randomString(3)
new_steward, new_steward_wallet, new_node = addNewStewardAndNode(looper,
steward,
steward_wallet,
new_steward_name,
node_name,
tdir,
client_tdir,
tconf,
all_plugins_path)
nodes.append(new_node)
looper.run(checkNodesConnected(nodes, customTimeout=60))
timeout = waits.expectedPoolCatchupTime(nodeCount=len(nodes))
waitNodeDataEquality(looper, new_node, *nodes[:-1],
customTimeout=timeout)
return new_node
def nodeThetaAdded(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1,
stewardWallet, allPluginsPath):
newStewardName = "testClientSteward" + randomString(3)
newNodeName = "Theta"
newSteward, newStewardWallet, newNode = addNewStewardAndNode(looper,
steward1,
stewardWallet,
newStewardName,
newNodeName,
tdirWithPoolTxns,
tconf,
allPluginsPath)
txnPoolNodeSet.append(newNode)
looper.run(checkNodesConnected(txnPoolNodeSet))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
*txnPoolNodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward,
*txnPoolNodeSet)
return newSteward, newStewardWallet, newNode
def test_upper_bound_of_checkpoint_after_catchup_is_divisible_by_chk_freq(
chkFreqPatched, looper, txnPoolNodeSet, steward1, stewardWallet,
client1, wallet1, tdir, client_tdir, tconf, allPluginsPath):
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
_, _, epsilon = addNewStewardAndNode(looper, steward1, stewardWallet,
'EpsilonSteward', 'Epsilon', tdir,
client_tdir, tconf, allPluginsPath)
txnPoolNodeSet.append(epsilon)
looper.run(checkNodesConnected(txnPoolNodeSet))
waitNodeDataEquality(looper, epsilon, *txnPoolNodeSet[:-1])
# Epsilon did not participate in ordering of the batch with EpsilonSteward
# NYM transaction and the batch with Epsilon NODE transaction.
# Epsilon got these transactions via catch-up.
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1)
for replica in epsilon.replicas:
assert len(replica.checkpoints) == 1
assert next(iter(replica.checkpoints)) == (7, 10)
def nodeCreatedAfterSomeTxns(
txnPoolNodesLooper, txnPoolNodeSet, tdirWithPoolTxns, poolTxnStewardData, tconf, allPluginsPath, request
):
# with Looper(debug=True) as looper:
client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient)
txnPoolNodesLooper.add(client)
txnPoolNodesLooper.run(client.ensureConnectedToNodes())
txnCount = getValueFromModule(request, "txnCount", 5)
sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, wallet, client, txnCount, timeoutPerReq=25)
newStewardName = randomString()
newNodeName = "Epsilon"
newStewardClient, newStewardWallet, newNode = addNewStewardAndNode(
txnPoolNodesLooper,
client,
wallet,
newStewardName,
newNodeName,
tdirWithPoolTxns,
tconf,
allPluginsPath=allPluginsPath,
autoStart=True,
)
yield txnPoolNodesLooper, newNode, client, wallet, newStewardClient, newStewardWallet
