def reqAcked1(looper, nodeSet, client1, sent1, faultyNodes):
coros = [partial(checkLastClientReqForNode, node, sent1)
for node in nodeSet]
looper.run(eventuallyAll(*coros,
totalTimeout=10,
acceptableFails=faultyNodes))
coros2 = [partial(checkReqAck, client1, node, sent1.identifier, sent1.reqId)
for node in nodeSet]
looper.run(eventuallyAll(*coros2,
totalTimeout=5,
acceptableFails=faultyNodes))
return sent1
def checkIfSameReplicaIsPrimary(looper: Looper,
replicas: Sequence[TestReplica] = None,
retryWait: float = 1,
timeout: float = 20):
# One and only one primary should be found and every replica should agree
# on same primary
def checkElectionDone():
unknowns = [r for r in replicas if r.primaryName is None]
assert len(unknowns) == 0, "election should be complete, " \
"but {} out of {} ({}) don't know who the primary " \
"is for protocol instance {}". \
format(len(unknowns), len(replicas), unknowns, replicas[0].instId)
def checkPrisAreOne(): # number of expected primaries
pris = sum(1 for r in replicas if r.isPrimary)
assert pris == 1, "Primary count should be 1, but was {} for " \
"protocol no {}".format(pris, replicas[0].instId)
def checkPrisAreSame():
pris = {r.primaryName for r in replicas}
assert len(pris) == 1, "Primary should be same for all, but were {} " \
"for protocol no {}" \
.format(pris, replicas[0].instId)
looper.run(
eventuallyAll(checkElectionDone, checkPrisAreOne, checkPrisAreSame,
retryWait=retryWait, totalTimeout=timeout))
def checkPropagated(looper, nodeSet, request, faultyNodes=0, timeout=10):
nodesSize = len(list(nodeSet.nodes))
# noinspection PyIncorrectDocstring
def g(node: TestNode):
"""
1. no of propagate received by node must be n -1 with zero
faulty nodes in system; where n = num of nodes
2. no of propagate received by node must be greater than
or equal to f + 1
"""
actualMsgs = len([
x for x in getAllArgs(node, Node.processPropagate)
if x['msg'].request[f.REQ_ID.nm] == request.reqId
and x['msg'].request[f.IDENTIFIER.nm] == request.identifier
and x['msg'].request[OPERATION] == request.operation
])
numOfMsgsWithZFN = nodesSize - 1
numOfMsgsWithFaults = faultyNodes + 1
assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN,
numOfMsgsWithFaults)
coros = [partial(g, node) for node in nodeSet]
looper.run(
eventuallyAll(*coros,
totalTimeout=timeout,
acceptableFails=faultyNodes))
def checkIfSameReplicaIsPrimary(looper: Looper,
replicas: Sequence[TestReplica] = None,
retryWait: float = 1,
timeout: float = 20):
# One and only one primary should be found and every replica should agree
# on same primary
def checkElectionDone():
unknowns = [r for r in replicas if r.primaryName is None]
assert len(unknowns) == 0, "election should be complete, " \
"but {} out of {} ({}) don't know who the primary " \
"is for protocol instance {}". \
format(len(unknowns), len(replicas), unknowns, replicas[0].instId)
def checkPrisAreOne(): # number of expected primaries
pris = sum(1 for r in replicas if r.isPrimary)
assert pris == 1, "Primary count should be 1, but was {} for " \
"protocol no {}".format(pris, replicas[0].instId)
def checkPrisAreSame():
pris = {r.primaryName for r in replicas}
assert len(pris) == 1, "Primary should be same for all, but were {} " \
"for protocol no {}" \
.format(pris, replicas[0].instId)
looper.run(
eventuallyAll(checkElectionDone,
checkPrisAreOne,
checkPrisAreSame,
retryWait=retryWait,
totalTimeout=timeout))
def wait_for_requests_ordered(looper, nodes, requests):
node_count = len(nodes)
timeout_per_request = waits.expectedTransactionExecutionTime(node_count)
total_timeout = (1 + len(requests) / 10) * timeout_per_request
coros = [partial(check_request_ordered,
node,
request)
for (node, request) in list(itertools.product(nodes, requests))]
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=total_timeout))
def check_reqacks(client, looper, reqs, txnPoolNodeSet):
reqack_coros = []
for req in reqs:
reqack_coros.extend([
partial(checkReqAck, client, node, req.identifier, req.reqId, None)
for node in txnPoolNodeSet
])
timeout = waits.expectedReqAckQuorumTime()
looper.run(eventuallyAll(*reqack_coros, totalTimeout=timeout))
def wait_for_requests_ordered(looper, nodes, requests):
node_count = len(nodes)
timeout_per_request = waits.expectedTransactionExecutionTime(node_count)
total_timeout = (1 + len(requests) / 10) * timeout_per_request
coros = [
partial(check_request_ordered, node, request)
for (node, request) in list(itertools.product(nodes, requests))
]
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=total_timeout))
def sdk_check_request_is_not_returned_to_nodes(looper, nodeSet, request):
instances = range(getNoInstances(len(nodeSet)))
coros = []
for node, inst_id in itertools.product(nodeSet, instances):
c = partial(checkRequestNotReturnedToNode,
node=node,
identifier=request['identifier'],
reqId=request['reqId'],
instId=inst_id)
coros.append(c)
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def sdk_check_request_is_not_returned_to_nodes(looper, nodeSet, request):
instances = range(getNoInstances(len(nodeSet)))
coros = []
for node, inst_id in itertools.product(nodeSet, instances):
c = partial(checkRequestNotReturnedToNode,
node=node,
identifier=request['identifier'],
reqId=request['reqId'],
instId=inst_id
)
coros.append(c)
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def testRequestFullRoundTrip(restrictiveVerifier,
client1,
sent1,
looper,
txnPoolNodeSet):
update = {'reason': 'client request invalid: InvalidClientRequest() '
'[caused by amount too high\nassert 999 <= 100]'}
coros2 = [partial(checkReqNack, client1, node, sent1.identifier,
sent1.reqId, update)
for node in txnPoolNodeSet]
timeout = waits.expectedReqAckQuorumTime()
looper.run(eventuallyAll(*coros2, totalTimeout=timeout))
def checkSufficientRepliesForRequests(looper,
client,
requests,
fVal=None,
timeoutPerReq=None):
nodeCount = len(client.nodeReg)
fVal = fVal or getMaxFailures(nodeCount)
timeoutPerReq = timeoutPerReq or 5 * nodeCount
coros = []
for request in requests:
coros.append(
partial(checkSufficientRepliesRecvd, client.inBox, request.reqId,
fVal))
looper.run(
eventuallyAll(*coros,
retryWait=1,
totalTimeout=timeoutPerReq * len(requests)))
def testClientRetryRequestWhenReplyNotReceived(looper, txnPoolNodeSet, client1,
wallet1, tconf):
"""
A node say Alpha sends ACK but doesn't send REPLY. The client resends the
request and gets REPLY
"""
alpha = txnPoolNodeSet[0]
skipped = False
origTrans = alpha.transmitToClient
def skipReplyOnce(msg, remoteName):
nonlocal skipped
if isinstance(msg, Reply) and not skipped:
skipped = True
return
origTrans(msg, remoteName)
alpha.transmitToClient = skipReplyOnce
req = sendRandomRequest(wallet1, client1)
coros = [
partial(checkReqAck, client1, node, *req.key)
for node in txnPoolNodeSet
]
timeout = waits.expectedReqAckQuorumTime()
start = time.perf_counter()
looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=timeout))
idr, reqId = req.key
# Client should get only 3 replies till the retry timeout since one node
# is not sending any replies
wait_for_replies(looper,
client1,
idr,
reqId,
3,
custom_timeout=tconf.CLIENT_REPLY_TIMEOUT - 1)
end = time.perf_counter()
# Client should wait till the retry timeout but after that should
# get the reply from the remaining node
looper.runFor(tconf.CLIENT_REPLY_TIMEOUT - (end - start))
wait_for_replies(looper, client1, idr, reqId, 4)
def wait_for_elections_done_on_given_nodes(looper: Looper,
nodes: Iterable[Node],
num_of_instances: int,
timeout: float,
retry_wait: float=1.0):
"""
Wait for primary elections to be completed on all the replicas
of the given nodes.
"""
def check_num_of_replicas():
for node in nodes:
assert len(node.replicas) == num_of_instances
def verify_each_replica_knows_its_primary():
for node in nodes:
for inst_id, replica in node.replicas.items():
assert replica.hasPrimary
looper.run(eventuallyAll(check_num_of_replicas,
verify_each_replica_knows_its_primary,
totalTimeout=timeout,
retryWait=retry_wait))
def wait_for_elections_done_on_given_nodes(looper: Looper,
nodes: Iterable[Node],
num_of_instances: int,
timeout: float,
retry_wait: float=1.0):
"""
Wait for primary elections to be completed on all the replicas
of the given nodes.
"""
def check_num_of_replicas():
for node in nodes:
assert len(node.replicas) == num_of_instances
def verify_each_replica_knows_its_primary():
for node in nodes:
for inst_id, replica in node.replicas.items():
assert replica.hasPrimary
looper.run(eventuallyAll(check_num_of_replicas,
verify_each_replica_knows_its_primary,
totalTimeout=timeout,
retryWait=retry_wait))
def testClientRetryRequestWhenReplyNotReceived(looper, nodeSet, client1,
wallet1, tconf):
"""
A node say Alpha sends ACK but doesn't send REPLY. The connect resends the
request and gets REPLY
"""
alpha = nodeSet.Alpha
skipped = False
origTrans = alpha.transmitToClient
def skipReplyOnce(msg, remoteName):
nonlocal skipped
if isinstance(msg, Reply) and not skipped:
skipped = True
return
origTrans(msg, remoteName)
alpha.transmitToClient = skipReplyOnce
req = sendRandomRequest(wallet1, client1)
coros = [partial(checkReqAck, client1, node, *req.key) for node in nodeSet]
looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=3))
looper.run(
eventually(checkReplyCount,
client1,
*req.key,
3,
retryWait=1,
timeout=3))
looper.run(
eventually(checkReplyCount,
client1,
*req.key,
4,
retryWait=1,
timeout=tconf.CLIENT_REPLY_TIMEOUT + 5))
def checkPrepared(looper,
nodeSet,
preprepared1,
instIds,
faultyNodes=0,
timeout=30):
nodeCount = len(list(nodeSet.nodes))
f = getMaxFailures(nodeCount)
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primaryDontSendAnyPREPAREs():
"""
1. no of PREPARE sent by primary should be 0
"""
for r in allReplicas:
for param in getAllArgs(r, Replica.processPrepare):
sender = param['sender']
assert sender != primary.name
def allReplicasSeeCorrectNumberOfPREPAREs():
"""
1. no of PREPARE received by replicas must be n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f
for replica in allReplicas:
key = primary.viewNo, primary.lastPrePrepareSeqNo
if key in replica.prepares:
actualMsgs = len(replica.prepares[key].voters)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(allReplicas) - faultyNodes
def primaryReceivesCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by primary replica is n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
actualMsgs = len([
param for param in getAllArgs(primary, primary.processPrepare)
if (param['prepare'].instId, param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.lastPrePrepareSeqNo)
and param['sender'] != primary.name
])
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f - 1
assert msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults)
# TODO what if the primary is faulty?
def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by Non primary replica is n - 2 without
faults and 2f - 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 2
numOfMsgsWithFaults = (2 * f) - 1
for npr in nonPrimaryReplicas:
actualMsgs = len([
param for param in getAllArgs(npr, npr.processPrepare)
if (param['prepare'].instId, param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (
primary.instId, primary.viewNo,
primary.lastPrePrepareSeqNo)
])
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
# TODO how do we know if one of the faulty nodes is a primary or
# not?
primaryDontSendAnyPREPAREs()
allReplicasSeeCorrectNumberOfPREPAREs()
primaryReceivesCorrectNumberOfPREPAREs()
nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
coros = [partial(g, instId) for instId in instIds]
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir,
pluginVerPath):
plugin = PluginLoader(pluginVerPath)
plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION]))
commonError = "client request invalid: InvalidClientRequest()"
client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir)
req = submitOp(wallet1, client1, {
TXN_TYPE: "dummy",
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}
})
validTypes = ', '.join(plugin.validTxnTypes)
update = {
'reason':
makeReason(
commonError, "dummy is not a valid "
"transaction type, must be "
"one of {}".format(validTypes))
}
coros1 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
})
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
coros2 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: "some string"
})
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
coros3 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: -3
}
})
update = {
'reason':
makeReason(
commonError, "{} must be present and should be "
"a number greater than 0".format(AMOUNT))
}
coros4 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
timeout = waits.expectedReqAckQuorumTime()
looper.run(
eventuallyAll(*(coros1 + coros2 + coros3 + coros4),
totalTimeout=timeout))
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}
})
waitForSufficientRepliesForRequests(looper,
client1,
requests=[req],
fVal=1)
for n in nodeSet: # type: Node
opVerifier, = n.opVerifiers
assert opVerifier.count == 1
def testAuctionReqValidationPlugin(looper, nodeSet, wallet1, client1, tdir,
pluginVerPath):
# TODO: Test more cases
plugin = PluginLoader(pluginVerPath)
plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION]))
commonError = "client request invalid: InvalidClientRequest()"
allCoros = []
op = {TXN_TYPE: "dummy", DATA: {AMOUNT: 30}}
req = submitOp(wallet1, client1, op)
validTypes = ', '.join(plugin.validTxnTypes)
update = {
'reason':
makeReason(
commonError, "dummy is not a valid transaction "
"type, must be one of {}".format(validTypes))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {
TXN_TYPE: AUCTION_START,
}
req = submitOp(wallet1, client1, op)
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {
TXN_TYPE: PLACE_BID,
}
req = submitOp(wallet1, client1, op)
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {TXN_TYPE: PLACE_BID, DATA: "some string"}
req = submitOp(wallet1, client1, op)
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {TXN_TYPE: PLACE_BID, DATA: {AMOUNT: 453}}
req = submitOp(wallet1, client1, op)
update = {'reason': makeReason(commonError, "No id provided for auction")}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {TXN_TYPE: AUCTION_START, DATA: {}}
req = submitOp(wallet1, client1, op)
update = {'reason': makeReason(commonError, "No id provided for auction")}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
op = {TXN_TYPE: AUCTION_END, DATA: {}}
req = submitOp(wallet1, client1, op)
update = {'reason': makeReason(commonError, "No id provided for auction")}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
auctionId = str(uuid4())
op = {TXN_TYPE: PLACE_BID, DATA: {ID: auctionId, AMOUNT: -3}}
req = submitOp(wallet1, client1, op)
update = {
'reason':
makeReason(
commonError, "{} must be present and should be "
"a number greater than 0".format(AMOUNT))
}
allCoros += [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
timeout = waits.expectedReqAckQuorumTime()
looper.run(eventuallyAll(*allCoros, totalTimeout=timeout))
for n in nodeSet: # type: Node
opVerifier, = n.opVerifiers
assert opVerifier.count == 0
def checkPrePrepared(looper,
nodeSet,
propagated1,
instIds,
faultyNodes=0,
timeout=30):
nodesSize = len(list(nodeSet))
def g(instId):
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primarySeesCorrectNumberOfPREPREPAREs():
"""
no of PRE-PREPARE as seen by processPrePrepare
method for primary must be 0 with or without faults in system
"""
l1 = len([
param
for param in getAllArgs(primary, primary.processPrePrepare)
])
assert l1 == 0
def nonPrimarySeesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be 1; whn zero faulty nodes in system.
2. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be greater than or equal to 0;
with faults in system.
"""
expectedPrePrepareRequest = PrePrepare(instId, primary.viewNo,
primary.lastPrePrepareSeqNo,
propagated1.identifier,
propagated1.reqId,
propagated1.digest,
time.time())
passes = 0
for npr in nonPrimaryReplicas:
actualMsgs = len([
param for param in getAllArgs(npr, npr.processPrePrepare)
if (param['pp'][:-1],
param['sender']) == (expectedPrePrepareRequest[:-1],
primary.name)
])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += int(
msgCountOK(nodesSize, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
def primarySentsCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE sent by primary is 1 with or without
fault in system but, when primary is faulty no of sent PRE_PREPARE
will be zero and primary must be marked as malicious.
"""
actualMsgs = len([
param for param in getAllArgs(primary, primary.doPrePrepare)
if (param['reqDigest'].identifier, param['reqDigest'].reqId,
param['reqDigest'].digest) == (propagated1.identifier,
propagated1.reqId,
propagated1.digest)
])
numOfMsgsWithZFN = 1
# TODO: Considering, Primary is not faulty and will always send
# PRE-PREPARE. Write separate test for testing when Primary
# is faulty
assert msgCountOK(nodesSize, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithZFN)
def nonPrimaryReceivesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE received by non-primaries must be 1
with zero faults in system, and 0 faults in system.
"""
passes = 0
for npr in nonPrimaryReplicas:
l4 = len([
param for param in getAllArgs(npr, npr.addToPrePrepares)
if (param['pp'].identifier, param['pp'].reqId,
param['pp'].digest) == (propagated1.identifier,
propagated1.reqId,
propagated1.digest)
])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += msgCountOK(nodesSize, faultyNodes, l4,
numOfMsgsWithZFN, numOfMsgsWithFaults)
assert passes >= len(nonPrimaryReplicas) - faultyNodes
primarySeesCorrectNumberOfPREPREPAREs()
nonPrimarySeesCorrectNumberOfPREPREPAREs()
primarySentsCorrectNumberOfPREPREPAREs()
nonPrimaryReceivesCorrectNumberOfPREPREPAREs()
coros = [partial(g, instId) for instId in instIds]
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def checkCommitted(looper,
nodeSet,
prepared1,
instIds,
faultyNodes=0,
timeout=60):
nodeCount = len((list(nodeSet)))
f = getMaxFailures(nodeCount)
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primaryReplica = getPrimaryReplica(nodeSet, instId)
def replicasSeesCorrectNumOfCOMMITs():
"""
num of commit messages must be = n when zero fault;
n = num of nodes and greater than or equal to
2f + 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount
numOfMsgsWithFault = (2 * f) + 1
key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo)
for r in allReplicas:
if key in r.commits:
rcvdCommitRqst = r.commits[key]
actualMsgsReceived = len(rcvdCommitRqst.voters)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
numOfMsgsWithZFN, numOfMsgsWithFault))
assert passes >= len(allReplicas) - faultyNodes
def replicasReceivesCorrectNumberOfCOMMITs():
"""
num of commit messages seen by replica must be equal to n - 1;
when zero fault and greater than or equal to
2f+1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFault = 2 * f
for r in allReplicas:
args = getAllArgs(r, r.processCommit)
actualMsgsReceived = len(args)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
numOfMsgsWithZFN, numOfMsgsWithFault))
for arg in args:
assert arg['commit'].viewNo == primaryReplica.viewNo and \
arg['commit'].ppSeqNo == primaryReplica.lastPrePrepareSeqNo and \
arg['commit'].digest == prepared1.digest
assert r.name != arg['sender']
assert passes >= len(allReplicas) - faultyNodes
replicasReceivesCorrectNumberOfCOMMITs()
replicasSeesCorrectNumOfCOMMITs()
coros = [partial(g, instId) for instId in instIds]
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
