def test_non_trust_anchor_cannot_add_attribute_for_user(nodeSet, nonTrustAnchor, trustAnchor, addedTrustAnchor, userIdA,
looper, attributeData):
with whitelistextras('UnauthorizedClientRequest'):
client, wallet = nonTrustAnchor
createNym(looper,
wallet.defaultId,
trustAnchor,
addedTrustAnchor,
role=None,
verkey=wallet.getVerkey())
attrib = Attribute(name='test1 attribute',
origin=wallet.defaultId,
value=attributeData,
dest=userIdA,
ledgerStore=LedgerStore.RAW)
reqs = makeAttribRequest(client, wallet, attrib)
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(checkRejects,
client,
reqs[0].reqId,
"UnauthorizedClientRequest('Only identity "
"owner/guardian can add attribute for that identity'",
retryWait=1, timeout=timeout))
def checkIdentityRequestSucceed(looper, actingClient, actingWallet, idr):
def chk():
assert actingWallet.getTrustAnchoredIdentity(idr).seqNo is not None
timeout = waits.expectedTransactionExecutionTime(
len(actingClient.nodeReg)
)
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def nymsAddedInQuickSuccession(nodeSet, addedTrustAnchor, looper,
trustAnchor, trustAnchorWallet):
usigner = DidSigner()
nym = usigner.verkey
idy = Identity(identifier=nym)
trustAnchorWallet.addTrustAnchoredIdentity(idy)
# Creating a NYM request with same nym again
req = idy.ledgerRequest()
trustAnchorWallet._pending.appendleft((req, idy.identifier))
reqs = trustAnchorWallet.preparePending()
trustAnchor.submitReqs(*reqs)
def check():
assert trustAnchorWallet._trustAnchored[nym].seqNo
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(check, timeout=timeout))
timeout = waits.expectedReqNAckQuorumTime()
looper.run(eventually(checkNacks,
trustAnchor,
req.reqId,
"is already added",
retryWait=1, timeout=timeout))
count = 0
for node in nodeSet:
for seq, txn in node.domainLedger.getAllTxn():
if txn[TXN_TYPE] == NYM and txn[TARGET_NYM] == usigner.identifier:
count += 1
assert(count == len(nodeSet))
def testPrePrepareDigest(setup, looper, sent1):
"""
A primary replica sends PRE-PREPARE message with incorrect digest to the
non primary replicas but non primary replicas should raise suspicion on
encountering the PRE-PREPARE. Also it should send no PREPARE
"""
primaryRep, nonPrimaryReps = setup.primaryRep, setup.nonPrimaryReps
def chkSusp():
for r in nonPrimaryReps:
# Every node with non primary replicas of instance 0 should raise
# suspicion
susp_code = Suspicions.PPR_DIGEST_WRONG.code
# Since the node sending bad requests might become primary of
# some backup instance after view changes, it will again send a
# PRE-PREPARE with incorrect digest, so other nodes might raise
# suspicion more than once
assert len(getNodeSuspicions(r.node,
susp_code)) >= 1
# No non primary replica should send any PREPARE
assert len(sentPrepare(r, viewNo=0, ppSeqNo=1)) == 0
numOfNodes = len(primaryRep.node.nodeReg)
timeout = waits.expectedTransactionExecutionTime(numOfNodes)
looper.run(eventually(chkSusp, retryWait=1, timeout=timeout))
def testMultiplePrePrepareWithSameSeqNo(setup, looper, sent1):
"""
A primary replica sends duplicate PRE-PREPARE messages to the non primary
replicas but non primary replicas should raise suspicion on encountering
each duplicate PRE-PREPARE. Also it should send only one PREPARE
"""
primaryRep, nonPrimaryReps = setup.primaryRep, setup.nonPrimaryReps
def chkSusp():
for r in nonPrimaryReps:
# Every node with non primary replicas of instance 0 should raise
# suspicion twice, once for each extra PRE-PREPARE request
suspectingNodes = \
getNodeSuspicions(r.node,
Suspicions.DUPLICATE_PPR_SENT.code)
assert len(suspectingNodes) == 2
# Each non primary replica should just send one PREPARE
assert len(sentPrepare(r)) == 1
numOfNodes = len(primaryRep.node.nodeReg)
timeout = waits.expectedTransactionExecutionTime(numOfNodes)
looper.run(eventually(chkSusp, retryWait=1, timeout=timeout))
def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, reqs_for_checkpoint):
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
max_batch_size = chkFreqPatched.Max3PCBatchSize
# Send some requests (insufficient for checkpoint),
# wait replies and check that current checkpoint is not stable
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2 * max_batch_size)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size, 2)
# From the steward send a request creating a user with None role
sdk_wallet_user = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 1, 3)
# From the created user send a request creating another user.
# Dynamic validation of this request must fail since a user with None role cannot create users.
# However, the 3PC-batch with the sent request must be ordered.
with pytest.raises(RequestRejectedException):
sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_user)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 2, 4)
# Send more requests to cause checkpoint stabilization
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, max_batch_size)
# Check that checkpoint is stable now
# and verify that requests for it were removed
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 0, 0)
# Send more requests to cause new checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint + 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 2, 0, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 1, 1)
def testMultipleInstanceChangeMsgsMarkNodeAsSuspicious(looper, txnPoolNodeSet):
maliciousNode = txnPoolNodeSet[0]
for i in range(0, 5):
maliciousNode.send(maliciousNode.view_changer._create_instance_change_msg(i, 0))
def chk(instId):
for node in txnPoolNodeSet:
if node.name != maliciousNode.name:
args = getAllArgs(node, ViewChanger.process_instance_change_msg)
assert len(args) == 5
for arg in args:
assert arg['frm'] == maliciousNode.name
numOfNodes = len(txnPoolNodeSet)
instanceChangeTimeout = waits.expectedPoolViewChangeStartedTimeout(
numOfNodes)
for i in range(0, 5):
looper.run(eventually(chk, i, retryWait=1,
timeout=instanceChangeTimeout))
def g():
for node in txnPoolNodeSet:
if node.name != maliciousNode.name:
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == maliciousNode.name
assert isinstance(reason, SuspiciousNode)
suspectingNodes = \
getNodeSuspicions(node,
Suspicions.FREQUENT_INST_CHNG.code)
assert len(suspectingNodes) == 13
timeout = waits.expectedTransactionExecutionTime(numOfNodes)
looper.run(eventually(g, retryWait=1, timeout=timeout))
def testPrepareDigest(setup, looper, sent1):
"""
A non primary replica sends PREPARE message with incorrect digest to all
other replicas. Other replicas should raise suspicion the
PREPARE seen
"""
primaryRep, nonPrimaryReps, faultyRep = setup.primaryRep, \
setup.nonPrimaryReps, \
setup.faultyRep
def chkSusp():
for r in (primaryRep, *nonPrimaryReps):
if r.name != faultyRep.name:
# Every node except the one from which PREPARE with incorrect
# digest was sent should raise suspicion for the PREPARE
# message
assert len(
getNodeSuspicions(
r.node,
Suspicions.PR_DIGEST_WRONG.code)) == 1
numOfNodes = len(primaryRep.node.nodeReg)
timeout = waits.expectedTransactionExecutionTime(numOfNodes)
looper.run(eventually(chkSusp, retryWait=1, timeout=timeout))
def test_lagged_checkpoint_completion(chkFreqPatched, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
One node in a pool lags to order the last 3PC-batch in a checkpoint so that
when it eventually orders this 3PC-batch and thus completes the checkpoint
it has already received and stashed the corresponding checkpoint messages
from all the other nodes. The test verifies that the node successfully
processes the stashed checkpoint messages and stabilizes the checkpoint.
"""
slow_node = txnPoolNodeSet[-1]
# All the nodes in the pool normally orders all the 3PC-batches in a
# checkpoint except the last 3PC-batch. The last 3PC-batch in the
# checkpoint is ordered by all the nodes except one slow node because this
# node lags to receive Commits.
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 4)
slow_node.nodeIbStasher.delay(cDelay())
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 1)
# All the other nodes complete the checkpoint and send Checkpoint messages
# to others. The slow node receives and stashes these messages because it
# has not completed the checkpoint.
def check():
for replica in slow_node.replicas.values():
assert len(replica.checkpoints) == 1
assert (1, 5) in replica.checkpoints
assert replica.checkpoints[(1, 5)].seqNo == 4
assert replica.checkpoints[(1, 5)].digest is None
assert replica.checkpoints[(1, 5)].isStable is False
assert len(replica.stashedRecvdCheckpoints) == 1
assert 0 in replica.stashedRecvdCheckpoints
assert len(replica.stashedRecvdCheckpoints[0]) == 1
assert (1, 5) in replica.stashedRecvdCheckpoints[0]
assert len(replica.stashedRecvdCheckpoints[0][(1, 5)]) == \
len(txnPoolNodeSet) - 1
stabilization_timeout = \
waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.run(eventually(check, timeout=stabilization_timeout))
# Eventually the slow node receives Commits, orders the last 3PC-batch in
# the checkpoint and thus completes it, processes the stashed checkpoint
# messages and stabilizes the checkpoint.
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
looper.runFor(waits.expectedOrderingTime(len(txnPoolNodeSet)))
for replica in slow_node.replicas.values():
assert len(replica.checkpoints) == 1
assert (1, 5) in replica.checkpoints
assert replica.checkpoints[(1, 5)].seqNo == 5
assert replica.checkpoints[(1, 5)].digest is not None
assert replica.checkpoints[(1, 5)].isStable is True
assert len(replica.stashedRecvdCheckpoints) == 0
def test_primary_recvs_3phase_message_outside_watermarks(perf_chk_patched, chkFreqPatched, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, reqs_for_logsize):
"""
One of the primary starts getting lot of requests, more than his log size
and queues up requests since they will go beyond its watermarks. This
happens since other nodes are slow in processing its PRE-PREPARE.
Eventually this primary will send PRE-PREPARE for all requests and those
requests will complete
"""
tconf = perf_chk_patched
delay = 2
instId = 0
reqs_to_send = 2 * reqs_for_logsize + 1
logger.debug('Will send {} requests'.format(reqs_to_send))
npr = getNonPrimaryReplicas(txnPoolNodeSet, instId)
pr = getPrimaryReplica(txnPoolNodeSet, instId)
orderedCount = pr.stats.get(TPCStat.OrderSent)
for r in npr:
r.node.nodeIbStasher.delay(ppDelay(delay, instId))
r.node.nodeIbStasher.delay(pDelay(delay, instId))
tm_exec_1_batch = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
batch_count = math.ceil(reqs_to_send / tconf.Max3PCBatchSize)
total_timeout = (tm_exec_1_batch + delay) * batch_count
def chk():
assert orderedCount + batch_count == pr.stats.get(TPCStat.OrderSent)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_to_send)
looper.run(eventually(chk, retryWait=1, timeout=total_timeout))
def wait_for_requests_ordered(looper, nodes, requests):
node_count = len(nodes)
timeout_per_request = waits.expectedTransactionExecutionTime(node_count)
total_timeout = (1 + len(requests) / 10) * timeout_per_request
coros = [partial(check_request_ordered,
node,
request)
for (node, request) in list(itertools.product(nodes, requests))]
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=total_timeout))
def testClientGetsResponseWithoutConsensusForUsedReqId(
nodeSet,
looper,
steward,
addedTrustAnchor,
trustAnchor,
userWalletA,
attributeName,
attributeData,
addedRawAttribute):
lastReqId = None
replies = {}
for msg, sender in reversed(trustAnchor.inBox):
if msg[OP_FIELD_NAME] == REPLY:
if not lastReqId:
lastReqId = get_req_id(msg[f.RESULT.nm])
if get_req_id(msg[f.RESULT.nm]) == lastReqId:
replies[sender] = msg
if len(replies) == len(nodeSet):
break
trustAnchorWallet = addedTrustAnchor
attrib = Attribute(name=attributeName,
origin=trustAnchorWallet.defaultId,
value=attributeData,
dest=userWalletA.defaultId,
ledgerStore=LedgerStore.RAW)
trustAnchorWallet.addAttribute(attrib)
req = trustAnchorWallet.preparePending()[0]
_, key = trustAnchorWallet._prepared.pop((req.identifier, req.reqId))
req.reqId = lastReqId
req.signature = trustAnchorWallet.signMsg(
msg=req.signingState(identifier=req.identifier),
identifier=req.identifier)
trustAnchorWallet._prepared[req.identifier, req.reqId] = req, key
trustAnchor.submitReqs(req)
def chk():
nonlocal trustAnchor, lastReqId, replies
for node in nodeSet:
last = node.spylog.getLast(TestNode.getReplyFromLedger.__name__)
assert last
result = last.result
assert result is not None
replies[node.clientstack.name][f.RESULT.nm].pop(TXN_TIME, None)
result.result.pop(TXN_TIME, None)
assert {k: v for k, v in result.result.items() if v is not None}.items() <= \
replies[node.clientstack.name][f.RESULT.nm].items()
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def ensureNymAdded(cli, nym, role=None):
ensureConnectedToTestEnv(cli)
cmd = "send NYM {dest}={nym}".format(dest=TARGET_NYM, nym=nym)
if role:
cmd += " {ROLE}={role}".format(ROLE=ROLE, role=role)
cli.enterCmd(cmd)
timeout = waits.expectedTransactionExecutionTime(len(cli.nodeReg))
cli.looper.run(
eventually(chkNymAddedOutput, cli, nym, retryWait=1, timeout=timeout))
timeout = waits.expectedTransactionExecutionTime(len(cli.nodeReg))
cli.enterCmd("send GET_NYM {dest}={nym}".format(dest=TARGET_NYM, nym=nym))
cli.looper.run(eventually(checkGetNym, cli, nym, retryWait=1, timeout=timeout))
cli.enterCmd('send ATTRIB {dest}={nym} raw={raw}'.
format(dest=TARGET_NYM, nym=nym,
# raw='{\"attrName\":\"attrValue\"}'))
raw=json.dumps({"attrName": "attrValue"})))
timeout = waits.expectedTransactionExecutionTime(len(cli.nodeReg))
cli.looper.run(eventually(checkAddAttr, cli, retryWait=1, timeout=timeout))
def sdk_eval_timeout(req_count: int, node_count: int,
customTimeoutPerReq: float = None, add_delay_to_timeout: float = 0):
timeout_per_request = customTimeoutPerReq or waits.expectedTransactionExecutionTime(node_count)
timeout_per_request += add_delay_to_timeout
# here we try to take into account what timeout for execution
# N request - total_timeout should be in
# timeout_per_request < total_timeout < timeout_per_request * N
# we cannot just take (timeout_per_request * N) because it is so huge.
# (for timeout_per_request=5 and N=10, total_timeout=50sec)
# lets start with some simple formula:
return (1 + req_count / 10) * timeout_per_request
def test_old_checkpoint_deleted(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint):
"""
Send requests more than twice of `CHK_FREQ`, there should be one new stable
checkpoint on each replica. The old stable checkpoint should be removed
"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2 * reqs_for_checkpoint)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 2, 0, retryWait=1, timeout=timeout))
def test_stable_checkpoint_when_one_instance_slow(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, reqs_for_checkpoint):
delay = 5
pr = getPrimaryReplica(txnPoolNodeSet, 1)
slowNode = pr.node
otherNodes = [n for n in txnPoolNodeSet if n != slowNode]
for n in otherNodes:
n.nodeIbStasher.delay(ppDelay(delay, 1))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint)
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + delay
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1, timeout=timeout))
def nodeThetaAdded(looper, nodeSet, tdirWithPoolTxns, tconf, steward,
stewardWallet, allPluginsPath, testNodeClass,
testClientClass, tdir):
newStewardName = "testClientSteward" + randomString(3)
newNodeName = "Theta"
newSteward, newStewardWallet = getClientAddedWithRole(nodeSet, tdir,
looper, steward,
stewardWallet,
newStewardName,
role=STEWARD)
sigseed = randomString(32).encode()
nodeSigner = SimpleSigner(seed=sigseed)
(nodeIp, nodePort), (clientIp, clientPort) = genHa(2)
data = {
NODE_IP: nodeIp,
NODE_PORT: nodePort,
CLIENT_IP: clientIp,
CLIENT_PORT: clientPort,
ALIAS: newNodeName,
SERVICES: [VALIDATOR, ]
}
node = Node(nodeSigner.identifier, data, newStewardWallet.defaultId)
newStewardWallet.addNode(node)
reqs = newStewardWallet.preparePending()
req, = newSteward.submitReqs(*reqs)
waitForSufficientRepliesForRequests(looper, newSteward, requests=[req])
def chk():
assert newStewardWallet.getNode(node.id).seqNo is not None
timeout = plenumWaits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
initNodeKeysForBothStacks(newNodeName, tdirWithPoolTxns, sigseed, override=True)
newNode = testNodeClass(newNodeName, basedirpath=tdir, config=tconf,
ha=(nodeIp, nodePort), cliha=(clientIp, clientPort),
pluginPaths=allPluginsPath)
nodeSet.append(newNode)
looper.add(newNode)
looper.run(checkNodesConnected(nodeSet))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward,
*nodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward,
*nodeSet)
return newSteward, newStewardWallet, newNode
async def send(self, op, org=None):
org = org if org else self.actor
req = org.client.submit(op)[0]
timeout = plenumWaits.expectedTransactionExecutionTime(
len(self.nodes))
for node in self.nodes:
await eventually(checkLastClientReqForNode,
node,
req,
retryWait=1,
timeout=timeout)
return req
def testClientGetsResponseWithoutConsensusForUsedReqId(nodeSet, looper, steward,
addedTrustAnchor, trustAnchor,
userWalletA,
attributeName,
attributeData,
addedRawAttribute):
lastReqId = None
replies = {}
for msg, sender in reversed(trustAnchor.inBox):
if msg[OP_FIELD_NAME] == REPLY:
if not lastReqId:
lastReqId = msg[f.RESULT.nm][f.REQ_ID.nm]
if msg.get(f.RESULT.nm, {}).get(f.REQ_ID.nm) == lastReqId:
replies[sender] = msg
if len(replies) == len(nodeSet):
break
trustAnchorWallet = addedTrustAnchor
attrib = Attribute(name=attributeName,
origin=trustAnchorWallet.defaultId,
value=attributeData,
dest=userWalletA.defaultId,
ledgerStore=LedgerStore.RAW)
trustAnchorWallet.addAttribute(attrib)
req = trustAnchorWallet.preparePending()[0]
_, key = trustAnchorWallet._prepared.pop((req.identifier, req.reqId))
req.reqId = lastReqId
req.signature = trustAnchorWallet.signMsg(msg=req.signingState,
identifier=req.identifier)
trustAnchorWallet._prepared[req.identifier, req.reqId] = req, key
trustAnchor.submitReqs(req)
def chk():
nonlocal trustAnchor, lastReqId, replies
for node in nodeSet:
last = node.spylog.getLast(TestNode.getReplyFromLedger.__name__)
assert last
result = last.result
assert result is not None
# TODO: Time is not equal as some precision is lost while storing
# in oientdb, using seconds may be an option, need to think of a
# use cases where time in milliseconds is required
replies[node.clientstack.name][f.RESULT.nm].pop(TXN_TIME, None)
result.result.pop(TXN_TIME, None)
assert replies[node.clientstack.name][f.RESULT.nm] == \
{k:v for k, v in result.result.items() if v is not None}
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def addAttributeAndCheck(looper, client, wallet, attrib):
old = wallet.pendingCount
pending = wallet.addAttribute(attrib)
assert pending == old + 1
reqs = wallet.preparePending()
client.submitReqs(*reqs)
def chk():
assert wallet.getAttribute(attrib).seqNo is not None
timeout = plenumWaits.expectedTransactionExecutionTime(client.totalNodes)
looper.run(eventually(chk, retryWait=1, timeout=timeout))
return wallet.getAttribute(attrib).seqNo
def testNonTrustAnchoredNymCanDoGetNym(nodeSet, addedTrustAnchor,
trustAnchorWallet, tdir, looper):
signer = DidSigner()
someClient, _ = genTestClient(nodeSet, tmpdir=tdir, usePoolLedger=True)
wallet = Wallet(someClient.name)
wallet.addIdentifier(signer=signer)
someClient.registerObserver(wallet.handleIncomingReply)
looper.add(someClient)
looper.run(someClient.ensureConnectedToNodes())
needle = trustAnchorWallet.defaultId
makeGetNymRequest(someClient, wallet, needle)
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(someClient.hasNym, needle, retryWait=1, timeout=timeout))
def test_repeated_request_not_processed_if_already_ordered(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
delta = txnPoolNodeSet[3]
initial_ledger_size = delta.domainLedger.size
one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1)
sdk_send_and_check(one_req, looper, txnPoolNodeSet, sdk_pool_handle)
sdk_send_signed_requests(sdk_pool_handle, one_req)
looper.runFor(waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)))
for node in txnPoolNodeSet:
assert node.domainLedger.size - initial_ledger_size == 1
def sdk_check_request_is_not_returned_to_nodes(looper, nodeSet, request):
instances = range(getNoInstances(len(nodeSet)))
coros = []
for node, inst_id in itertools.product(nodeSet, instances):
c = partial(checkRequestNotReturnedToNode,
node=node,
identifier=request['identifier'],
reqId=request['reqId'],
instId=inst_id
)
coros.append(c)
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def test_node_erases_last_sent_pp_key_on_view_change(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf):
# Get a node with a backup primary replica
replica = getPrimaryReplica(txnPoolNodeSet, instId=backup_inst_id)
node = replica.node
# Send some 3PC-batches and wait until the replica orders the 3PC-batches
sdk_send_batches_of_random(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
num_reqs=3, num_batches=3,
timeout=tconf.Max3PCBatchWait)
looper.run(
eventually(lambda: assertExp(replica.last_ordered_3pc == (0, 3)),
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Ensure that there is a stored last sent PrePrepare key on the node
assert LAST_SENT_PRE_PREPARE in node.nodeStatusDB
# Make the pool perform view change
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper, txnPoolNodeSet)
# Verify that the node has erased the stored last sent PrePrepare key
assert LAST_SENT_PRE_PREPARE not in node.nodeStatusDB
# Send a 3PC-batch and ensure that the replica orders it
sdk_send_batches_of_random(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
num_reqs=1, num_batches=1,
timeout=tconf.Max3PCBatchWait)
looper.run(
eventually(lambda: assertExp(replica.last_ordered_3pc == (1, 1)),
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
def sdk_get_reply(looper, sdk_req_resp, timeout=None):
req_json, resp_task = sdk_req_resp
# TODO: change timeout evaluating logic, when sdk will can tuning timeout from outside
if timeout is None:
timeout = waits.expectedTransactionExecutionTime(7)
try:
resp = looper.run(asyncio.wait_for(resp_task, timeout=timeout))
resp = json.loads(resp)
except IndyError as e:
resp = e.error_code
except TimeoutError as e:
resp = ErrorCode.PoolLedgerTimeout
return req_json, resp
def submitPoolUpgrade(looper, senderClient, senderWallet, name, action, version,
schedule, timeout, sha256):
upgrade = Upgrade(name, action, schedule, version, sha256, timeout,
senderWallet.defaultId)
senderWallet.doPoolUpgrade(upgrade)
reqs = senderWallet.preparePending()
senderClient.submitReqs(*reqs)
def check():
assert senderWallet._upgrades[upgrade.key].seqNo
timeout = waits.expectedTransactionExecutionTime(
len(senderClient.nodeReg)
)
looper.run(eventually(check, timeout=timeout))
def getAttribute(looper, trustAnchor, trustAnchorWallet, userIdA, attributeName,
attributeValue):
# Should be renamed to get_attribute_and_check
attrib = Attribute(name=attributeName,
value=None,
dest=userIdA,
ledgerStore=LedgerStore.RAW)
req = trustAnchorWallet.requestAttribute(attrib,
sender=trustAnchorWallet.defaultId)
trustAnchor.submitReqs(req)
timeout = waits.expectedTransactionExecutionTime(len(trustAnchor.nodeReg))
return looper.run(eventually(checkGetAttr, req.key, trustAnchor,
attributeName, attributeValue, retryWait=1,
timeout=timeout))
def test_belated_propagate_not_processed_if_already_ordered(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
delta = txnPoolNodeSet[3]
initial_ledger_size = delta.domainLedger.size
delta.nodeIbStasher.delay(ppgDelay(300, 'Gamma'))
one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1)
sdk_send_and_check(one_req, looper, txnPoolNodeSet, sdk_pool_handle)
delta.nodeIbStasher.reset_delays_and_process_delayeds()
looper.runFor(waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)))
for node in txnPoolNodeSet:
assert node.domainLedger.size - initial_ledger_size == 1
def createNym(looper, nym, creatorClient, creatorWallet: Wallet, role=None,
verkey=None):
idy = Identity(identifier=nym,
verkey=verkey,
role=role)
creatorWallet.addTrustAnchoredIdentity(idy)
reqs = creatorWallet.preparePending()
creatorClient.submitReqs(*reqs)
def check():
assert creatorWallet._trustAnchored[nym].seqNo
timeout = waits.expectedTransactionExecutionTime(
len(creatorClient.nodeReg)
)
looper.run(eventually(check, retryWait=1, timeout=timeout))
def testTrustyCancelsUpgrade(validUpgradeSent, looper, nodeSet, trustee,
trusteeWallet, validUpgrade):
validUpgradeCopy = deepcopy(validUpgrade)
validUpgradeCopy[ACTION] = CANCEL
validUpgradeCopy[JUSTIFICATION] = '"not gonna give you one"'
validUpgradeCopy.pop(SCHEDULE, None)
upgrade, req = sendUpgrade(trustee, trusteeWallet, validUpgradeCopy)
def check():
assert trusteeWallet.getPoolUpgrade(upgrade.key).seqNo
timeout = plenumWaits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(check, retryWait=1, timeout=timeout))
looper.run(eventually(checkNoUpgradeScheduled, nodeSet, retryWait=1,
timeout=waits.expectedNoUpgradeScheduled()))
def testLoggingTxnStateWhenCommitFails(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, logsearch):
logsPropagate, _ = logsearch(levels=['INFO'],
files=['propagator.py'],
funcs=['propagate'],
msgs=['propagating.*request.*from client'])
logsOrdered, _ = logsearch(levels=['INFO'],
files=['replica.py'],
funcs=['order_3pc_key'],
msgs=['ordered batch request'])
logsCommitFail, _ = logsearch(levels=['WARNING'],
files=['node.py'],
funcs=['executeBatch'],
msgs=['commit failed for batch request'])
seed = randomString(32)
wh, _ = sdk_wallet_steward
nym_request, _ = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_steward, seed, "name", None))
sdk_sign_and_send_prepared_request(looper, sdk_wallet_steward,
sdk_pool_handle, nym_request)
class SomeError(Exception):
pass
def commitPatched(node, commitOrig, *args, **kwargs):
req_handler = node.get_req_handler(ledger_id=DOMAIN_LEDGER_ID)
req_handler.commit = commitOrig
raise SomeError(ERORR_MSG)
excCounter = 0
def executeBatchPatched(node, executeBatchOrig, *args, **kwargs):
nonlocal excCounter
try:
executeBatchOrig(*args, **kwargs)
except SomeError:
excCounter += 1
node.executeBatch = executeBatchOrig
pass
def checkSufficientExceptionsHappend():
assert excCounter == len(txnPoolNodeSet)
return
for node in txnPoolNodeSet:
req_handler = node.get_req_handler(ledger_id=DOMAIN_LEDGER_ID)
req_handler.commit = functools.partial(commitPatched, node,
req_handler.commit)
node.executeBatch = functools.partial(executeBatchPatched, node,
node.executeBatch)
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.run(
eventually(checkSufficientExceptionsHappend,
retryWait=1,
timeout=timeout))
reqId = str(json.loads(nym_request)['reqId'])
assert any(reqId in record.getMessage() for record in logsPropagate)
assert any(reqId in record.getMessage() for record in logsOrdered)
assert any(reqId in record.getMessage() for record in logsCommitFail)
assert any(ERORR_MSG in record.getMessage() for record in logsCommitFail)
def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
"""
Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
number "n" if it has already accepted a request with view number "v" and
sequence number "n"
"""
numOfNodes = 4
fValue = getMaxFailures(numOfNodes)
primaryRepl = getPrimaryReplica(txnPoolNodeSet, 1)
logger.debug("Primary Replica: {}".format(primaryRepl))
nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, 1)
logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
# Delay COMMITs so request is not ordered and checks can be made
c_delay = 10
for node in txnPoolNodeSet:
node.nodeIbStasher.delay(cDelay(delay=c_delay, instId=1))
req1 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
1)[0]
request1 = sdk_json_to_request_object(req1[0])
for npr in nonPrimaryReplicas:
looper.run(
eventually(checkPrepareReqSent,
npr,
request1.key,
primaryRepl.viewNo,
retryWait=1))
prePrepareReq = primaryRepl._ordering_service.sent_preprepares[
primaryRepl.viewNo, primaryRepl.lastPrePrepareSeqNo]
looper.run(
eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1))
# logger.debug("Patching the primary replica's pre-prepare sending method ")
# orig_method = primaryRepl.sendPrePrepare
# def patched(self, ppReq):
# self._ordering_service.sent_preprepares[ppReq.viewNo, ppReq.ppSeqNo] = ppReq
# ppReq = updateNamedTuple(ppReq, **{f.PP_SEQ_NO.nm: 1})
# self.send(ppReq, TPCStat.PrePrepareSent)
#
# primaryRepl.sendPrePrepare = types.MethodType(patched, primaryRepl)
logger.debug(
"Decrementing the primary replica's pre-prepare sequence number by "
"one...")
primaryRepl._ordering_service._lastPrePrepareSeqNo -= 1
view_no = primaryRepl.viewNo
request2 = sdk_json_to_request_object(
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
1)[0][0])
timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet))
looper.run(
eventually(checkPrePrepareReqSent,
primaryRepl,
request2,
retryWait=1,
timeout=timeout))
# Since the node is malicious, it will not be able to process requests due
# to conflicts in PRE-PREPARE
primaryRepl.node.stop()
looper.removeProdable(primaryRepl.node)
reqIdr = [request2.digest]
prePrepareReq = PrePrepare(
primaryRepl.instId, view_no, primaryRepl.lastPrePrepareSeqNo,
get_utc_epoch(), reqIdr, init_discarded(),
primaryRepl.batchDigest([request2]), DOMAIN_LEDGER_ID,
primaryRepl._ordering_service.get_state_root_hash(DOMAIN_LEDGER_ID),
primaryRepl._ordering_service.get_txn_root_hash(DOMAIN_LEDGER_ID), 0,
True, [])
logger.debug("""Checking whether all the non primary replicas have received
the pre-prepare request with same sequence number""")
timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet))
looper.run(
eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1,
timeout=timeout))
logger.debug("""Check that none of the non primary replicas didn't send
any prepare message "
in response to the pre-prepare message""")
timeout = waits.expectedPrepareTime(len(txnPoolNodeSet))
looper.runFor(timeout) # expect prepare processing timeout
# check if prepares have not been sent
for npr in nonPrimaryReplicas:
with pytest.raises(AssertionError):
looper.run(
eventually(checkPrepareReqSent,
npr,
request2.key,
view_no,
retryWait=1,
timeout=timeout))
timeout = waits.expectedTransactionExecutionTime(
len(txnPoolNodeSet)) + c_delay
result1 = sdk_get_replies(looper, [req1])[0][1]
logger.debug("request {} gives result {}".format(request1, result1))
def test_stashed_messages_processed_on_backup_replica_ordering_resumption(
looper, chkFreqPatched, reqs_for_checkpoint,
one_replica_and_others_in_backup_instance,
sdk_pool_handle, sdk_wallet_client, view_change_done,
txnPoolNodeSet):
"""
Verifies resumption of ordering 3PC-batches on a backup replica
on detection of a lag in checkpoints in case it is detected after
some 3PC-messages related to the next checkpoint have already been stashed
as laying outside of the watermarks.
Please note that to verify this case the config is set up so that
LOG_SIZE == (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ
"""
global first_run
batches_count = get_pp_seq_no(txnPoolNodeSet)
slow_replica, other_replicas = one_replica_and_others_in_backup_instance
view_no = slow_replica.viewNo
low_watermark = slow_replica.h
# Send a request and ensure that the replica orders the batch for it
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
batches_count += 1
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc == (view_no, batches_count)),
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Don't receive Commits from two replicas
slow_replica.node.nodeIbStasher.delay(
cDelay(instId=1, sender_filter=other_replicas[0].node.name))
slow_replica.node.nodeIbStasher.delay(
cDelay(instId=1, sender_filter=other_replicas[1].node.name))
slow_replica.node.nodeIbStasher.delay(
msg_rep_delay(types_to_delay=[COMMIT])
)
# Send a request for which the replica will not be able to order the batch
# due to an insufficient count of Commits
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Receive further Commits from now on
slow_replica.node.nodeIbStasher.drop_delayeds()
slow_replica.node.nodeIbStasher.resetDelays()
# Send requests but in a quantity insufficient
# for catch-up number of checkpoints
reqs_until_checkpoints = reqs_for_checkpoint - get_pp_seq_no([r.node for r in other_replicas]) % reqs_for_checkpoint
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP *
reqs_until_checkpoints)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Don't receive Checkpoints
slow_replica.node.nodeIbStasher.delay(chk_delay(instId=1))
# Send more requests to reach catch-up number of checkpoints
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
reqs_for_checkpoint)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Ensure that there are no 3PC-messages stashed
# as laying outside of the watermarks
assert slow_replica.stasher.stash_size(STASH_WATERMARKS) == 0
# Send a request for which the batch will be outside of the watermarks
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Ensure that the replica has not ordered any batches
# after the very first one
assert slow_replica.last_ordered_3pc == (view_no, batches_count)
# Ensure that the watermarks have not been shifted since the view start
assert slow_replica.h == low_watermark
assert slow_replica.H == (sys.maxsize if first_run else low_watermark + LOG_SIZE)
# Ensure that there are some quorumed stashed checkpoints
check_num_quorumed_received_checkpoints(slow_replica, 1)
# Ensure that now there are 3PC-messages stashed
# as laying outside of the watermarks
if not first_run:
assert slow_replica.stasher.stash_size(STASH_WATERMARKS) == incoming_3pc_msgs_count(len(txnPoolNodeSet))
# Receive belated Checkpoints
slow_replica.node.nodeIbStasher.reset_delays_and_process_delayeds()
batches_count = get_pp_seq_no([r.node for r in other_replicas])
# Ensure that the replica has ordered the batch for the last sent request
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc ==
(view_no, batches_count)),
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Ensure that the watermarks have been shifted so that the lower watermark
# now equals to the end of the last stable checkpoint in the instance
assert slow_replica.h == low_watermark + (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ
assert slow_replica.H == low_watermark + (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ + LOG_SIZE
# Ensure that now there are no quorumed stashed checkpoints
check_num_quorumed_received_checkpoints(slow_replica, 0)
# Ensure that now there are no 3PC-messages stashed
# as laying outside of the watermarks
assert slow_replica.stasher.stash_size(STASH_WATERMARKS) == 0
# Send a request and ensure that the replica orders the batch for it
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
batches_count += 1
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc ==
(view_no, batches_count)),
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
first_run = False
def test_backup_primary_restores_pp_seq_no_if_view_is_same(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf,
tdir, allPluginsPath, chkFreqPatched, view_no):
# Get a node with a backup primary replica
replica = getPrimaryReplica(txnPoolNodeSet, instId=backup_inst_id)
batches_count = 0 if view_no == 0 else 1
node = replica.node
# Send some 3PC-batches and wait until the replica orders the 3PC-batches
sdk_send_batches_of_random(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=7,
num_batches=num_batches,
timeout=tconf.Max3PCBatchWait)
batches_count += num_batches
looper.run(
eventually(lambda r: assertExp(r.last_ordered_3pc ==
(view_no, batches_count)),
replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Check view no of the node and lastPrePrepareSeqNo of the replica
assert node.viewNo == view_no
assert replica.lastPrePrepareSeqNo == batches_count
# Ensure that the node has stored the last sent PrePrepare key
assert LAST_SENT_PRE_PREPARE in node.nodeStatusDB
last_sent_pre_prepare_key = \
node_status_db_serializer.deserialize(
node.nodeStatusDB.get(LAST_SENT_PRE_PREPARE))
assert last_sent_pre_prepare_key == {
str(backup_inst_id): [view_no, batches_count]
}
# Restart the node containing the replica
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
node.name,
stopNode=True)
looper.removeProdable(node)
txnPoolNodeSet.remove(node)
node = start_stopped_node(node, looper, tconf, tdir, allPluginsPath)
txnPoolNodeSet.append(node)
looper.run(checkNodesConnected(txnPoolNodeSet))
ensureElectionsDone(looper, txnPoolNodeSet)
replica = node.replicas[backup_inst_id]
# Verify that after the successful propagate primary procedure the replica
# (which must still be the primary in its instance) has restored
# lastPrePrepareSeqNo and adjusted last_ordered_3pc and shifted
# the watermarks correspondingly
assert node.viewNo == view_no
assert replica.isPrimary
assert replica.lastPrePrepareSeqNo == batches_count
assert replica.last_ordered_3pc == (view_no, batches_count)
assert replica.h == batches_count
assert replica.H == batches_count + LOG_SIZE
# Verify also that the stored last sent PrePrepare key has not been erased
assert LAST_SENT_PRE_PREPARE in node.nodeStatusDB
# Send a 3PC-batch and ensure that the replica orders it
sdk_send_batches_of_random(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=1,
num_batches=1,
timeout=tconf.Max3PCBatchWait)
batches_count += 1
looper.run(
eventually(lambda: assertExp(replica.last_ordered_3pc ==
(view_no, batches_count)),
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
def test_second_checkpoint_after_catchup_can_be_stabilized(
chkFreqPatched, looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_wallet_client, sdk_pool_handle, tdir, tconf, allPluginsPath):
_, new_node = sdk_add_new_steward_and_node(looper,
sdk_pool_handle,
sdk_wallet_steward,
'EpsilonSteward',
'Epsilon',
tdir,
tconf,
allPluginsPath=allPluginsPath)
txnPoolNodeSet.append(new_node)
looper.run(checkNodesConnected(txnPoolNodeSet))
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
# Epsilon did not participate in ordering of the batch with EpsilonSteward
# NYM transaction and the batch with Epsilon NODE transaction.
# Epsilon got these transactions via catch-up.
master_replica = new_node.replicas._master_replica
assert len(master_replica.checkpoints) == 0
assert len(master_replica.stashedRecvdCheckpoints) == 0
assert master_replica.h == 2
assert master_replica.H == 17
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
for replica in new_node.replicas.values():
assert len(replica.checkpoints) == 1
assert len(replica.stashedRecvdCheckpoints) == 0
assert replica.h == 2
assert replica.H == 17
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 6)
stabilization_timeout = \
waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.runFor(stabilization_timeout)
for replica in new_node.replicas.values():
assert len(replica.checkpoints) == 2
keys_iter = iter(replica.checkpoints)
assert next(keys_iter) == (3, 5)
assert replica.checkpoints[3, 5].seqNo == 5
assert replica.checkpoints[3, 5].digest is None
assert replica.checkpoints[3, 5].isStable is False
assert next(keys_iter) == (6, 10)
assert replica.checkpoints[6, 10].seqNo == 9
assert replica.checkpoints[6, 10].digest is None
assert replica.checkpoints[6, 10].isStable is False
# nothing is stashed since it's ordered during catch-up
assert len(replica.stashedRecvdCheckpoints) == 0
assert replica.h == 2
assert replica.H == 17
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
looper.runFor(stabilization_timeout)
for replica in new_node.replicas.values():
assert len(replica.checkpoints) == 1
keys_iter = iter(replica.checkpoints)
assert next(keys_iter) == (6, 10)
assert replica.checkpoints[6, 10].seqNo == 10
assert replica.checkpoints[6, 10].digest is not None
assert replica.checkpoints[6, 10].isStable is True
assert len(replica.stashedRecvdCheckpoints) == 0
assert replica.h == 10
assert replica.H == 25
def waitForReply(cli, nodeCount, replyChecker, customTimeout=None):
timeout = customTimeout or \
waits.expectedTransactionExecutionTime(nodeCount)
cli.looper.run(
eventually(checkReply, cli, nodeCount, replyChecker, timeout=timeout))
def test_backup_replica_resumes_ordering_on_lag_in_checkpoints(
looper, chkFreqPatched, reqs_for_checkpoint,
one_replica_and_others_in_backup_instance,
sdk_pool_handle, sdk_wallet_client, view_change_done, txnPoolNodeSet):
"""
Verifies resumption of ordering 3PC-batches on a backup replica
on detection of a lag in checkpoints
"""
slow_replica, other_replicas = one_replica_and_others_in_backup_instance
view_no = slow_replica.viewNo
# Send a request and ensure that the replica orders the batch for it
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc == (view_no, 2)),
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Don't receive Commits from two replicas
slow_replica.node.nodeIbStasher.delay(
cDelay(instId=1, sender_filter=other_replicas[0].node.name))
slow_replica.node.nodeIbStasher.delay(
cDelay(instId=1, sender_filter=other_replicas[1].node.name))
# Send a request for which the replica will not be able to order the batch
# due to an insufficient count of Commits
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Recover reception of Commits
slow_replica.node.nodeIbStasher.drop_delayeds()
slow_replica.node.nodeIbStasher.resetDelays()
# Send requests but in a quantity insufficient
# for catch-up number of checkpoints
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP *
reqs_for_checkpoint - 3)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Ensure that the replica has not ordered any batches
# after the very first one
assert slow_replica.last_ordered_3pc == (view_no, 2)
# Ensure that the watermarks have not been shifted since the view start
assert slow_replica.h == 0
assert slow_replica.H == LOG_SIZE
# Ensure that the collections related to requests, batches and
# own checkpoints are not empty.
# (Note that a primary replica removes requests from requestQueues
# when creating a batch with them.)
if slow_replica.isPrimary:
assert slow_replica.sentPrePrepares
else:
assert slow_replica.requestQueues[DOMAIN_LEDGER_ID]
assert slow_replica.prePrepares
assert slow_replica.prepares
assert slow_replica.commits
assert slow_replica.batches
assert slow_replica._checkpointer._checkpoint_state
print(slow_replica._checkpointer._stashed_recvd_checkpoints)
# Ensure that there are some quorumed stashed checkpoints
assert slow_replica._checkpointer._stashed_checkpoints_with_quorum()
# Send more requests to reach catch-up number of checkpoints
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, reqs_for_checkpoint)
# Ensure that the replica has adjusted last_ordered_3pc to the end
# of the last checkpoint
def chk(r):
print(r)
assert r.last_ordered_3pc == (view_no, (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ)
looper.run(
eventually(chk,
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Ensure that the watermarks have been shifted so that the lower watermark
# has the same value as last_ordered_3pc
assert slow_replica.h == (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ
assert slow_replica.H == (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ + LOG_SIZE
# Ensure that the collections related to requests, batches and
# own checkpoints have been cleared
assert not slow_replica.requestQueues[DOMAIN_LEDGER_ID]
assert not slow_replica.sentPrePrepares
assert not slow_replica.prePrepares
assert not slow_replica.prepares
assert not slow_replica.commits
assert not slow_replica.batches
assert not slow_replica._checkpointer._checkpoint_state
# Ensure that now there are no quorumed stashed checkpoints
assert not slow_replica._checkpointer._stashed_checkpoints_with_quorum()
# Send a request and ensure that the replica orders the batch for it
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc ==
(view_no, (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ + 1)),
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
def testReplyMatchesRequest(looper, nodeSet, tdir, up):
'''
This tests does check following things:
- wallet works correctly when used by multiple clients
- clients do receive responses for exactly the same request they sent
'''
def makeClient(id):
client, wallet = genTestClient(nodeSet,
tmpdir=tdir,
name="client-{}".format(id))
looper.add(client)
looper.run(client.ensureConnectedToNodes())
return client, wallet
# creating clients
numOfClients = 3
numOfRequests = 1
clients = set()
sharedWallet = None
for i in range(numOfClients):
client, wallet = makeClient(i)
if sharedWallet is None:
sharedWallet = wallet
clients.add(client)
for i in range(1, numOfRequests + 1):
# sending requests
requests = {}
for client in clients:
op = randomOperation()
req = sharedWallet.signOp(op)
request = client.submitReqs(req)[0][0]
requests[client] = (request.reqId, request.operation['amount'])
# checking results
responseTimeout = waits.expectedTransactionExecutionTime(nodeCount)
for client, (reqId, sentAmount) in requests.items():
looper.run(
eventually(checkResponseRecvdFromNodes,
client,
nodeCount,
reqId,
retryWait=1,
timeout=responseTimeout))
print("Expected amount for request {} is {}".format(
reqId, sentAmount))
# This looks like it fails on some python versions
# replies = [r[0]['result']['amount']
# for r in client.inBox
# if r[0]['op'] == 'REPLY'
# and r[0]['result']['reqId'] == reqId]
replies = []
for r in client.inBox:
if r[0]['op'] == 'REPLY' and r[0]['result']['reqId'] == reqId:
if 'amount' not in r[0]['result']:
logger.debug('{} cannot find amount in {}'.format(
client, r[0]['result']))
replies.append(r[0]['result']['amount'])
assert all(replies[0] == r for r in replies)
assert replies[0] == sentAmount
def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
reqs_for_checkpoint):
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
max_batch_size = chkFreqPatched.Max3PCBatchSize
# Send some requests (insufficient for checkpoint),
# wait replies and check that current checkpoint is not stable
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 2 * max_batch_size)
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
0,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size, 2)
# From the steward send a request creating a user with None role
sdk_wallet_user = sdk_add_new_nym(looper, sdk_pool_handle,
sdk_wallet_steward)
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
0,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 1, 3)
# From the created user send a request creating another user.
# Dynamic validation of this request must fail since a user with None role cannot create users.
# However, the 3PC-batch with the sent request must be ordered.
with pytest.raises(RequestRejectedException):
sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_user)
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
0,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 2, 4)
# Send more requests to cause checkpoint stabilization
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, max_batch_size)
# Check that checkpoint is stable now
# and verify that requests for it were removed
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
5,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 0, 0)
# Send more requests to cause new checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, reqs_for_checkpoint + 1)
looper.run(
eventually(check_for_nodes,
txnPoolNodeSet,
check_stable_checkpoint,
10,
retryWait=1,
timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 1, 1)
def test_stashed_checkpoint_processing(chkFreqPatched, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
One node in a pool of 5 nodes lags to order the last 3PC-batch in a
checkpoint. By the moment when it eventually orders the 3PC-batch it has
already received and stashed Checkpoint message from two node, so it
processes these stashed messages on completing the checkpoint. After this
it receives Checkpoint messages from two other nodes, processes them and
stabilizes the checkpoint.
"""
epsilon = txnPoolNodeSet[-1]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 4)
epsilon.nodeIbStasher.delay(cDelay())
epsilon.nodeIbStasher.delay(chk_delay(sender_filter='Gamma'))
epsilon.nodeIbStasher.delay(chk_delay(sender_filter='Delta'))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
stabilization_timeout = \
waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.runFor(stabilization_timeout)
for inst_id, replica in epsilon.replicas:
assert len(replica.checkpoints) == 1
assert (1, 5) in replica.checkpoints
assert replica.checkpoints[(1, 5)].seqNo == 4
assert replica.checkpoints[(1, 5)].digest is None
assert replica.checkpoints[(1, 5)].isStable is False
assert len(replica.stashedRecvdCheckpoints) == 1
assert 0 in replica.stashedRecvdCheckpoints
assert len(replica.stashedRecvdCheckpoints[0]) == 1
assert (1, 5) in replica.stashedRecvdCheckpoints[0]
assert len(replica.stashedRecvdCheckpoints[0][(1, 5)]) == 2
epsilon.nodeIbStasher.reset_delays_and_process_delayeds(COMMIT)
def check():
for inst_id, replica in epsilon.replicas:
assert len(replica.checkpoints) == 1
assert (1, 5) in replica.checkpoints
assert replica.checkpoints[(1, 5)].seqNo == 5
assert replica.checkpoints[(1, 5)].digest is not None
assert replica.checkpoints[(1, 5)].isStable is False
assert len(replica.stashedRecvdCheckpoints) == 0
looper.run(
eventually(check,
timeout=waits.expectedOrderingTime(len(txnPoolNodeSet))))
epsilon.nodeIbStasher.reset_delays_and_process_delayeds(CHECKPOINT)
stabilization_timeout = \
waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.runFor(stabilization_timeout)
for inst_id, replica in epsilon.replicas:
assert len(replica.checkpoints) == 1
assert (1, 5) in replica.checkpoints
assert replica.checkpoints[(1, 5)].seqNo == 5
assert replica.checkpoints[(1, 5)].digest is not None
assert replica.checkpoints[(1, 5)].isStable is True
assert len(replica.stashedRecvdCheckpoints) == 0
def test_non_primary_recvs_3phase_message_outside_watermarks(
chkFreqPatched, reqs_for_logsize, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
A node is slow in receiving PRE-PREPAREs and PREPAREs. A lot of requests
are sent and the slow node has started receiving COMMITs outside of its
watermarks and so stashes them. Also this node is slow in receiving
CHECKPOINTs. So a catch-up does not occur on it.
Then the slow node eventually receives the sent PRE-PREPAREs and PREPAREs
and so orders the 3PC-batches between its watermarks. The other nodes
discard the COMMITs from the slow node since they have already achieved
stable checkpoints for these COMMITs.
After that the slow node eventually receives the sent CHECKPOINTs from
the other nodes and so stabilizes own completed checkpoints and updates its
watermarks. A catch-up is not triggered because no received checkpoints are
stashed. Since now the watermarks have been updated, the slow node
processes 3PC-messages stashed earlier and its ledger becomes equal to the
ledgers of the other nodes.
"""
backupInstId = 1
npr = getNonPrimaryReplicas(txnPoolNodeSet, backupInstId)
slowReplica = npr[0]
slowNode = slowReplica.node
slowNode.nodeIbStasher.delay(ppDelay(300, backupInstId))
slowNode.nodeIbStasher.delay(pDelay(300, backupInstId))
slowNode.nodeIbStasher.delay(chk_delay(300))
initialDomainLedgerSize = slowNode.domainLedger.size
oldStashCount = slowReplica.stasher.num_stashed_watermarks
slowReplica.H = LOG_SIZE
# 1. Send requests more than fit between the watermarks on the slow node
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_logsize + 2)
# Verify that the slow node stashes the batches outside of its watermarks
newStashCount = slowReplica.stasher.num_stashed_watermarks
assert newStashCount > oldStashCount
oldDiscardCounts = discardCounts([n.replicas[backupInstId] for n in txnPoolNodeSet if n != slowNode],
'achieved stable checkpoint')
# 2. Deliver the sent PREPREPAREs and PREPAREs to the slow node
slowNode.nodeIbStasher.reset_delays_and_process_delayeds(PREPREPARE, PREPARE)
# Verify that the slow node orders the 3PC-batches between its watermarks
# but no more.
looper.runFor(waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)))
checkNodeDataForInequality(slowNode, *[n for n in txnPoolNodeSet if n != slowNode])
assert slowNode.domainLedger.size - initialDomainLedgerSize == reqs_for_logsize
# Also verify that the other nodes discard the COMMITs from the slow node
# since they have already achieved stable checkpoints for these COMMITs.
counts = discardCounts(
[n.replicas[backupInstId] for n in txnPoolNodeSet if n != slowNode],
'achieved stable checkpoint')
for nm, count in counts.items():
assert count > oldDiscardCounts[nm]
oldCatchupTimes = slowNode.spylog.count(Node.start_catchup)
# 3. Deliver the sent CHECKPOINTs to the slow node
slowNode.nodeIbStasher.reset_delays_and_process_delayeds(CHECKPOINT)
# Verify that the slow node processes 3PC-messages stashed earlier and its
# ledger becomes equal to the ledgers of the other nodes while a catch-up
# is not made.
waitNodeDataEquality(looper, slowNode, *[n for n in txnPoolNodeSet if n != slowNode])
assert slowNode.domainLedger.size - initialDomainLedgerSize == reqs_for_logsize + 2
newCatchupTimes = slowNode.spylog.count(Node.start_catchup)
assert newCatchupTimes == oldCatchupTimes
def wait_for_replies(looper, client, idr, reqId, count, custom_timeout=None):
timeout = custom_timeout or waits.expectedTransactionExecutionTime(
len(client.nodeReg))
looper.run(
eventually(checkReplyCount, client, idr, reqId, count,
timeout=timeout))
def nodeThetaAdded(looper,
nodeSet,
tdirWithClientPoolTxns,
tconf,
steward,
stewardWallet,
allPluginsPath,
testNodeClass,
testClientClass,
node_config_helper_class,
tdir,
node_name='Theta'):
newStewardName = "testClientSteward" + randomString(3)
newNodeName = node_name
newSteward, newStewardWallet = getClientAddedWithRole(
nodeSet,
tdirWithClientPoolTxns,
looper,
steward,
stewardWallet,
newStewardName,
role=STEWARD)
sigseed = randomString(32).encode()
nodeSigner = SimpleSigner(seed=sigseed)
(nodeIp, nodePort), (clientIp, clientPort) = genHa(2)
config_helper = node_config_helper_class(newNodeName, tconf, chroot=tdir)
_, _, bls_key = initNodeKeysForBothStacks(newNodeName,
config_helper.keys_dir,
sigseed,
override=True)
data = {
NODE_IP: nodeIp,
NODE_PORT: nodePort,
CLIENT_IP: clientIp,
CLIENT_PORT: clientPort,
ALIAS: newNodeName,
SERVICES: [
VALIDATOR,
],
BLS_KEY: bls_key
}
node = Node(nodeSigner.identifier, data, newStewardWallet.defaultId)
newStewardWallet.addNode(node)
reqs = newStewardWallet.preparePending()
req = newSteward.submitReqs(*reqs)[0][0]
waitForSufficientRepliesForRequests(looper, newSteward, requests=[req])
def chk():
assert newStewardWallet.getNode(node.id).seqNo is not None
timeout = plenumWaits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
newNode = testNodeClass(newNodeName,
config_helper=config_helper,
config=tconf,
ha=(nodeIp, nodePort),
cliha=(clientIp, clientPort),
pluginPaths=allPluginsPath)
nodeSet.append(newNode)
looper.add(newNode)
looper.run(checkNodesConnected(nodeSet))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward, *nodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *nodeSet)
return newSteward, newStewardWallet, newNode
def test_backup_replica_resumes_ordering_on_lag_if_checkpoints_belate(
looper, chkFreqPatched, reqs_for_checkpoint,
one_replica_and_others_in_backup_instance,
sdk_pool_handle, sdk_wallet_client, view_change_done):
"""
Verifies resumption of ordering 3PC-batches on a backup replica
on detection of a lag in checkpoints in case it is detected after
some batch in the next checkpoint has already been committed but cannot
be ordered out of turn
"""
slow_replica, other_replicas = one_replica_and_others_in_backup_instance
view_no = slow_replica.viewNo
# Send a request and ensure that the replica orders the batch for it
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc == (view_no, 2)),
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Don't receive Commits from two replicas
slow_replica.node.nodeIbStasher.delay(
cDelay(instId=1, sender_filter=other_replicas[0].node.name))
slow_replica.node.nodeIbStasher.delay(
cDelay(instId=1, sender_filter=other_replicas[1].node.name))
# Send a request for which the replica will not be able to order the batch
# due to an insufficient count of Commits
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Receive further Commits from now on
slow_replica.node.nodeIbStasher.drop_delayeds()
slow_replica.node.nodeIbStasher.resetDelays()
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc == (view_no, 2)),
slow_replica,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Send requests but in a quantity insufficient
# for catch-up number of checkpoints
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP *
reqs_for_checkpoint - 2)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Don't receive Checkpoints
slow_replica.node.nodeIbStasher.delay(chk_delay(instId=1))
# Send more requests to reach catch-up number of checkpoints
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
reqs_for_checkpoint)
# Send a request that starts a new checkpoint
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Ensure that the replica has not ordered any batches
# after the very first one
assert slow_replica.last_ordered_3pc == (view_no, 2)
# Ensure that the watermarks have not been shifted since the view start
assert slow_replica.h == 0
assert slow_replica.H == LOG_SIZE
# Ensure that there are some quorumed stashed checkpoints
assert slow_replica._checkpointer._stashed_checkpoints_with_quorum()
# Receive belated Checkpoints
slow_replica.node.nodeIbStasher.reset_delays_and_process_delayeds()
# Ensure that the replica has ordered the batch for the last sent request
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc == \
(view_no, (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ + 2)),
slow_replica,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Ensure that the watermarks have been shifted so that the lower watermark
# now equals to the end of the last stable checkpoint in the instance
assert slow_replica.h == (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ
assert slow_replica.H == (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ + LOG_SIZE
# Ensure that now there are no quorumed stashed checkpoints
assert not slow_replica._checkpointer._stashed_checkpoints_with_quorum()
# Send a request and ensure that the replica orders the batch for it
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.run(
eventually(lambda: assertExp(slow_replica.last_ordered_3pc ==
(view_no, (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ + 3)),
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
def testTransactions(cli, loadAuctionReqPlugin,
createAllNodes, validNodeNames):
nodeCount = len(validNodeNames)
auctionId = str(uuid4())
names = ["Jason", "John", "Les", "Timothy", "Tyler"]
for name in names:
createClientAndConnect(cli, validNodeNames, name)
jason, tyler, les, john, timothy = names
timeout = waits.expectedTransactionExecutionTime(nodeCount)
cli.enterCmd("client {} start auction {}".format(jason, auctionId))
waitRequestSuccess(cli, nodeCount * 1, customTimeout=timeout)
cli.enterCmd("client {} place bid 20 on {}".format(tyler, auctionId))
waitRequestSuccess(cli, nodeCount * 2, customTimeout=timeout)
cli.enterCmd("client {} balance".format(tyler))
waitRequestSuccess(cli, nodeCount * 3, customTimeout=timeout)
waitBalanceChange(cli, nodeCount, 980, customTimeout=timeout)
cli.enterCmd("client {} place bid 40 on {}".format(les, auctionId))
waitRequestSuccess(cli, nodeCount * 4, customTimeout=timeout)
cli.enterCmd("client {} balance".format(tyler))
waitRequestSuccess(cli, nodeCount * 5, customTimeout=timeout)
waitBalanceChange(cli, nodeCount, 1000, customTimeout=timeout)
cli.enterCmd("client {} balance".format(les))
waitRequestSuccess(cli, nodeCount * 6, customTimeout=timeout)
waitBalanceChange(cli, nodeCount, 960, customTimeout=timeout)
# This bid would fail so `success` would be false and thus success count
# wont increase
cli.enterCmd("client {} place bid 30 on {}".format(john, auctionId))
waitRequestSuccess(cli, nodeCount * 6, customTimeout=timeout)
cli.enterCmd("client {} balance".format(john))
waitRequestSuccess(cli, nodeCount * 7, customTimeout=timeout)
waitBalanceChange(cli, nodeCount * 2, 1000, customTimeout=timeout)
cli.enterCmd("client {} balance".format(les))
waitRequestSuccess(cli, nodeCount * 8, customTimeout=timeout)
waitBalanceChange(cli, nodeCount * 2, 960, customTimeout=timeout)
cli.enterCmd("client {} place bid 200 on {}".format(timothy, auctionId))
waitRequestSuccess(cli, nodeCount * 9, customTimeout=timeout)
cli.enterCmd("client {} balance".format(timothy))
waitRequestSuccess(cli, nodeCount * 10, customTimeout=timeout)
waitBalanceChange(cli, nodeCount, 800, customTimeout=timeout)
cli.enterCmd("client {} balance".format(les))
waitRequestSuccess(cli, nodeCount * 11, customTimeout=timeout)
waitBalanceChange(cli, nodeCount * 3, 1000, customTimeout=timeout)
cli.enterCmd("client {} end auction {}".format(jason, auctionId))
waitRequestSuccess(cli, nodeCount * 12, customTimeout=timeout)
cli.enterCmd("client {} place bid 300 on {}".format(john, auctionId))
waitRequestSuccess(cli, nodeCount * 12, customTimeout=timeout)
cli.enterCmd("client {} balance".format(john))
waitRequestSuccess(cli, nodeCount * 13, customTimeout=timeout)
waitBalanceChange(cli, nodeCount * 4, 1000, customTimeout=timeout)
cli.enterCmd("client {} balance".format(tyler))
waitRequestSuccess(cli, nodeCount * 14, customTimeout=timeout)
waitBalanceChange(cli, nodeCount * 5, 1000, customTimeout=timeout)
cli.enterCmd("client {} balance".format(john))
waitRequestSuccess(cli, nodeCount * 15, customTimeout=timeout)
waitBalanceChange(cli, nodeCount * 6, 1000, customTimeout=timeout)
cli.enterCmd("client {} balance".format(les))
waitRequestSuccess(cli, nodeCount * 16, customTimeout=timeout)
waitBalanceChange(cli, nodeCount * 7, 1000, customTimeout=timeout)
cli.enterCmd("client {} balance".format(timothy))
waitRequestSuccess(cli, nodeCount * 17, customTimeout=timeout)
waitBalanceChange(cli, nodeCount * 2, 800, customTimeout=timeout)
def checkIdentityRequestSucceed(looper, actingClient, actingWallet, idr):
def chk():
assert actingWallet.getTrustAnchoredIdentity(idr).seqNo is not None
timeout = waits.expectedTransactionExecutionTime(len(actingClient.nodeReg))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def test_node_erases_last_sent_pp_key_on_pool_restart(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client, tconf,
tdir, allPluginsPath,
chkFreqPatched):
# Get a node with a backup primary replica and the rest of the nodes
replica = getPrimaryReplica(txnPoolNodeSet, instId=backup_inst_id)
node = replica.node
# Send some 3PC-batches and wait until the replica orders the 3PC-batches
sdk_send_batches_of_random(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=7,
num_batches=7,
timeout=tconf.Max3PCBatchWait)
looper.run(
eventually(lambda: assertExp(replica.last_ordered_3pc == (0, 7)),
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Check view no of the node and lastPrePrepareSeqNo of the replica
assert node.viewNo == 0
assert replica.lastPrePrepareSeqNo == 7
assert replica.h == 6
assert replica.H == 6 + LOG_SIZE
# Ensure that there is a stored last sent PrePrepare key on the node
assert LAST_SENT_PRE_PREPARE in node.nodeStatusDB
# Restart all the nodes in the pool and wait for primary elections done
all_nodes = copy(txnPoolNodeSet)
for n in all_nodes:
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
n.name,
timeout=nodeCount,
stopNode=True)
looper.removeProdable(n)
txnPoolNodeSet.remove(n)
for n in all_nodes:
txnPoolNodeSet.append(
start_stopped_node(n, looper, tconf, tdir, allPluginsPath))
looper.run(checkNodesConnected(txnPoolNodeSet))
ensureElectionsDone(looper, txnPoolNodeSet)
node = nodeByName(txnPoolNodeSet, node.name)
replica = node.replicas[backup_inst_id]
# Verify that the node has erased the stored last sent PrePrepare key
assert LAST_SENT_PRE_PREPARE not in node.nodeStatusDB
# Verify correspondingly that after the pool restart the replica
# (which must again be the primary in its instance) has not restored
# lastPrePrepareSeqNo, not adjusted last_ordered_3pc and not shifted
# the watermarks
assert node.viewNo == 0
assert replica.isPrimary
assert replica.lastPrePrepareSeqNo == 0
assert replica.last_ordered_3pc == (0, 0)
assert replica.h == 0
assert replica.H == 0 + LOG_SIZE
# Send a 3PC-batch and ensure that the replica orders it
sdk_send_batches_of_random(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=1,
num_batches=1,
timeout=tconf.Max3PCBatchWait)
looper.run(
eventually(lambda: assertExp(replica.last_ordered_3pc == (0, 1)),
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
