def reqAcked1(looper, nodeSet, client1, sent1, faultyNodes):
coros = [
partial(checkLastClientReqForNode, node, sent1) for node in nodeSet
]
looper.run(
eventuallyAll(*coros, totalTimeout=10, acceptableFails=faultyNodes))
coros2 = [
partial(checkReqAck, client1, node, sent1.identifier, sent1.reqId)
for node in nodeSet
]
looper.run(
eventuallyAll(*coros2, totalTimeout=5, acceptableFails=faultyNodes))
return sent1
def checkPropagated(looper, nodeSet, request, faultyNodes=0):
nodesSize = len(list(nodeSet.nodes))
# noinspection PyIncorrectDocstring
def g(node: TestNode):
"""
1. no of propagate received by node must be n -1 with zero
faulty nodes in system; where n = num of nodes
2. no of propagate received by node must be greater than
or equal to f + 1
"""
actualMsgs = len([
x for x in getAllArgs(node, Node.processPropagate)
if x['msg'].request[f.REQ_ID.nm] == request.reqId
and x['msg'].request[f.IDENTIFIER.nm] == request.identifier
and x['msg'].request[OPERATION] == request.operation
])
numOfMsgsWithZFN = nodesSize - 1
numOfMsgsWithFaults = faultyNodes + 1
assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN,
numOfMsgsWithFaults)
coros = [partial(g, node) for node in nodeSet]
looper.run(
eventuallyAll(*coros, totalTimeout=10, acceptableFails=faultyNodes))
def testClientRetryRequestWhenReplyNotReceived(looper, nodeSet, client1,
wallet1, tconf):
"""
A node say Alpha sends ACK but doesn't send REPLY. The connect resends the
request and gets REPLY
"""
alpha = nodeSet.Alpha
skipped = False
origTrans = alpha.transmitToClient
def skipReplyOnce(msg, remoteName):
nonlocal skipped
if isinstance(msg, Reply) and not skipped:
skipped = True
return
origTrans(msg, remoteName)
alpha.transmitToClient = skipReplyOnce
req = sendRandomRequest(wallet1, client1)
coros = [partial(checkReqAck, client1, node, *req.key) for node in nodeSet]
looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=3))
looper.run(eventually(checkReplyCount, client1, *req.key, 3, retryWait=1,
timeout=3))
looper.run(eventually(checkReplyCount, client1, *req.key, 4, retryWait=1,
timeout=tconf.CLIENT_REPLY_TIMEOUT + 5))
def checkIfSameReplicaIPrimary(looper: Looper,
replicas: Sequence[TestReplica] = None,
retryWait: float = 1,
timeout: float = 20):
# One and only one primary should be found and every replica should agree
# on same primary
def checkElectionDone():
unknowns = sum(1 for r in replicas if r.isPrimary is None)
assert unknowns == 0, "election should be complete, but {} out of {} " \
"don't know who the primary is for " \
"protocol no {}".\
format(unknowns, len(replicas), replicas[0].instId)
def checkPrisAreOne(): # number of expected primaries
pris = sum(1 for r in replicas if r.isPrimary)
assert pris == 1, "Primary count should be 1, but was {} for " \
"protocol no {}".format(pris, replicas[0].instId)
def checkPrisAreSame():
pris = {r.primaryName for r in replicas}
assert len(pris) == 1, "Primary should be same for all, but were {} " \
"for protocol no {}" \
.format(pris, replicas[0].instId)
looper.run(
eventuallyAll(checkElectionDone,
checkPrisAreOne,
checkPrisAreSame,
retryWait=retryWait,
totalTimeout=timeout))
def checkSufficientRepliesForRequests(looper, client, requests, fVal=None,
timeoutPerReq=None):
nodeCount = len(client.nodeReg)
fVal = fVal or getMaxFailures(nodeCount)
timeoutPerReq = timeoutPerReq or 5 * nodeCount
coros = []
for request in requests:
coros.append(partial(checkSufficientRepliesRecvd, client.inBox,
request.reqId, fVal))
looper.run(eventuallyAll(*coros, retryWait=1,
totalTimeout=timeoutPerReq * len(requests)))
def testRequestFullRoundTrip(restrictiveVerifier, client1, sent1, looper,
nodeSet):
update = {
'reason':
'client request invalid: InvalidClientRequest() '
'[caused by amount too high\nassert 999 <= 100]'
}
coros2 = [
partial(checkReqNack, client1, node, sent1.identifier, sent1.reqId,
update) for node in nodeSet
]
looper.run(eventuallyAll(*coros2, totalTimeout=5))
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir,
pluginVerPath):
plugin = PluginLoader(pluginVerPath)
plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION]))
commonError = "client request invalid: InvalidClientRequest()"
client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir)
req = submitOp(wallet1, client1, {
TXN_TYPE: "dummy",
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}
})
validTypes = ', '.join(plugin.validTxnTypes)
update = {
'reason':
makeReason(
commonError, "dummy is not a valid "
"transaction type, must be "
"one of {}".format(validTypes))
}
coros1 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
})
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
coros2 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: "some string"
})
update = {
'reason':
makeReason(
commonError,
"{} attribute is missing or not in proper format".format(DATA))
}
coros3 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: -3
}
})
update = {
'reason':
makeReason(
commonError, "{} must be present and should be "
"a number greater than 0".format(AMOUNT))
}
coros4 = [
partial(checkReqNack, client1, node, req.identifier, req.reqId, update)
for node in nodeSet
]
looper.run(
eventuallyAll(*(coros1 + coros2 + coros3 + coros4), totalTimeout=5))
req = submitOp(wallet1, client1, {
TXN_TYPE: CREDIT,
TARGET_NYM: wallet2.defaultId,
DATA: {
AMOUNT: 30
}
})
looper.run(
eventually(checkSufficientRepliesRecvd,
client1.inBox,
req.reqId,
1,
retryWait=1,
timeout=5))
for n in nodeSet: # type: Node
opVerifier, = n.opVerifiers
assert opVerifier.count == 1
def checkPrePrepared(looper, nodeSet, propagated1, instIds, faultyNodes=0):
nodesSize = len(list(nodeSet))
def g(instId):
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primarySeesCorrectNumberOfPREPREPAREs():
"""
no of PRE-PREPARE as seen by processPrePrepare
method for primary must be 0 with or without faults in system
"""
l1 = len([
param
for param in getAllArgs(primary, primary.processPrePrepare)
])
assert l1 == 0
def nonPrimarySeesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be 1; whn zero faulty nodes in system.
2. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be greater than or equal to 0;
with faults in system.
"""
expectedPrePrepareRequest = PrePrepare(instId, primary.viewNo,
primary.lastPrePrepareSeqNo,
propagated1.identifier,
propagated1.reqId,
propagated1.digest,
time.time())
passes = 0
for npr in nonPrimaryReplicas:
actualMsgs = len([
param for param in getAllArgs(npr, npr.processPrePrepare)
if (param['pp'][:-1],
param['sender']) == (expectedPrePrepareRequest[:-1],
primary.name)
])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += int(
msgCountOK(nodesSize, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
def primarySentsCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE sent by primary is 1 with or without
fault in system but, when primary is faulty no of sent PRE_PREPARE
will be zero and primary must be marked as malicious.
"""
actualMsgs = len([
param for param in getAllArgs(primary, primary.doPrePrepare)
if (param['reqDigest'].identifier, param['reqDigest'].reqId,
param['reqDigest'].digest) == (propagated1.identifier,
propagated1.reqId,
propagated1.digest)
])
numOfMsgsWithZFN = 1
# TODO: Considering, Primary is not faulty and will always send
# PRE-PREPARE. Write separate test for testing when Primary
# is faulty
assert msgCountOK(nodesSize, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithZFN)
def nonPrimaryReceivesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE received by non-primaries must be 1
with zero faults in system, and 0 faults in system.
"""
passes = 0
for npr in nonPrimaryReplicas:
l4 = len([
param for param in getAllArgs(npr, npr.addToPrePrepares)
if (param['pp'].identifier, param['pp'].reqId,
param['pp'].digest) == (propagated1.identifier,
propagated1.reqId,
propagated1.digest)
])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += msgCountOK(nodesSize, faultyNodes, l4,
numOfMsgsWithZFN, numOfMsgsWithFaults)
assert passes >= len(nonPrimaryReplicas) - faultyNodes
primarySeesCorrectNumberOfPREPREPAREs()
nonPrimarySeesCorrectNumberOfPREPREPAREs()
primarySentsCorrectNumberOfPREPREPAREs()
nonPrimaryReceivesCorrectNumberOfPREPREPAREs()
coros = [partial(g, instId) for instId in instIds]
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=30))
def checkCommited(looper, nodeSet, prepared1, instIds, faultyNodes=0):
nodeCount = len((list(nodeSet)))
f = getMaxFailures(nodeCount)
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primaryReplica = getPrimaryReplica(nodeSet, instId)
def replicasSeesCorrectNumOfCOMMITs():
"""
num of commit messages must be = n when zero fault;
n = num of nodes and greater than or equal to
2f + 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount
numOfMsgsWithFault = (2 * f) + 1
key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo)
for r in allReplicas:
if key in r.commits:
rcvdCommitRqst = r.commits[key]
actualMsgsReceived = len(rcvdCommitRqst.voters)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
numOfMsgsWithZFN, numOfMsgsWithFault))
assert passes >= len(allReplicas) - faultyNodes
def replicasReceivesCorrectNumberOfCOMMITs():
"""
num of commit messages seen by replica must be equal to n - 1;
when zero fault and greater than or equal to
2f+1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFault = 2 * f
for r in allReplicas:
args = getAllArgs(r, r.processCommit)
actualMsgsReceived = len(args)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
numOfMsgsWithZFN, numOfMsgsWithFault))
for arg in args:
assert arg['commit'].viewNo == primaryReplica.viewNo and \
arg['commit'].ppSeqNo == primaryReplica.lastPrePrepareSeqNo and \
arg['commit'].digest == prepared1.digest
assert r.name != arg['sender']
assert passes >= len(allReplicas) - faultyNodes
replicasReceivesCorrectNumberOfCOMMITs()
replicasSeesCorrectNumOfCOMMITs()
coros = [partial(g, instId) for instId in instIds]
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=60))
def checkPrepared(looper, nodeSet, preprepared1, instIds, faultyNodes=0):
nodeCount = len(list(nodeSet.nodes))
f = getMaxFailures(nodeCount)
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primaryDontSendAnyPREPAREs():
"""
1. no of PREPARE sent by primary should be 0
"""
for r in allReplicas:
for param in getAllArgs(r, Replica.processPrepare):
sender = param['sender']
assert sender != primary.name
def allReplicasSeeCorrectNumberOfPREPAREs():
"""
1. no of PREPARE received by replicas must be n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f
for replica in allReplicas:
key = primary.viewNo, primary.lastPrePrepareSeqNo
if key in replica.prepares:
actualMsgs = len(replica.prepares[key].voters)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(allReplicas) - faultyNodes
def primaryReceivesCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by primary replica is n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
actualMsgs = len([
param for param in getAllArgs(primary, primary.processPrepare)
if (param['prepare'].instId, param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.lastPrePrepareSeqNo)
and param['sender'] != primary.name
])
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f - 1
assert msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults)
# TODO what if the primary is faulty?
def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by Non primary replica is n - 2 without
faults and 2f - 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 2
numOfMsgsWithFaults = (2 * f) - 1
for npr in nonPrimaryReplicas:
actualMsgs = len([
param for param in getAllArgs(npr, npr.processPrepare)
if (param['prepare'].instId, param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (
primary.instId, primary.viewNo,
primary.lastPrePrepareSeqNo)
])
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
# TODO how do we know if one of the faulty nodes is a primary or
# not?
primaryDontSendAnyPREPAREs()
allReplicasSeeCorrectNumberOfPREPAREs()
primaryReceivesCorrectNumberOfPREPAREs()
nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
coros = [partial(g, instId) for instId in instIds]
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=30))
def testAuctionReqValidationPlugin(looper, nodeSet, wallet1, client1, tdir,
pluginVerPath):
# TODO: Test more cases
plugin = PluginLoader(pluginVerPath)
plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION]))
commonError = "client request invalid: InvalidClientRequest()"
allCoros = []
op = {
TXN_TYPE: "dummy",
DATA: {
AMOUNT: 30
}}
req = submitOp(wallet1, client1, op)
validTypes = ', '.join(plugin.validTxnTypes)
update = {
'reason': makeReason(commonError, "dummy is not a valid transaction "
"type, must be one of {}"
.format(validTypes))}
allCoros += [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
op = {
TXN_TYPE: AUCTION_START,
}
req = submitOp(wallet1, client1, op)
update = {
'reason': makeReason(commonError,
"{} attribute is missing or not in proper format"
.format(DATA))}
allCoros += [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
op = {
TXN_TYPE: PLACE_BID,
}
req = submitOp(wallet1, client1, op)
update = {
'reason': makeReason(commonError,
"{} attribute is missing or not in proper format"
.format(DATA))}
allCoros += [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
op = {
TXN_TYPE: PLACE_BID,
DATA: "some string"
}
req = submitOp(wallet1, client1, op)
update = {
'reason': makeReason(commonError,
"{} attribute is missing or not in proper format"
.format(DATA))}
allCoros += [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
op = {
TXN_TYPE: PLACE_BID,
DATA: {
AMOUNT: 453
}}
req = submitOp(wallet1, client1, op)
update = {
'reason': makeReason(commonError, "No id provided for auction")}
allCoros += [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
op = {
TXN_TYPE: AUCTION_START,
DATA: {}
}
req = submitOp(wallet1, client1, op)
update = {
'reason': makeReason(commonError, "No id provided for auction")}
allCoros += [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
op = {
TXN_TYPE: AUCTION_END,
DATA: {}
}
req = submitOp(wallet1, client1, op)
update = {
'reason': makeReason(commonError, "No id provided for auction")}
allCoros += [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
auctionId = str(uuid4())
op = {
TXN_TYPE: PLACE_BID,
DATA: {
ID: auctionId,
AMOUNT: -3
}
}
req = submitOp(wallet1, client1, op)
update = {
'reason': makeReason(commonError, "{} must be present and should be "
"a number greater than 0"
.format(AMOUNT))}
allCoros += [partial(checkReqNack, client1, node, req.identifier,
req.reqId, update) for node in nodeSet]
looper.run(eventuallyAll(*allCoros, totalTimeout=5))
for n in nodeSet: # type: Node
opVerifier, = n.opVerifiers
assert opVerifier.count == 0