def reqAcked1(looper, nodeSet, client1, sent1, faultyNodes): coros = [partial(checkLastClientReqForNode, node, sent1) for node in nodeSet] looper.run(eventuallyAll(*coros, totalTimeout=10, acceptableFails=faultyNodes)) coros2 = [partial(checkReqAck, client1, node, sent1.identifier, sent1.reqId) for node in nodeSet] looper.run(eventuallyAll(*coros2, totalTimeout=5, acceptableFails=faultyNodes)) return sent1
def checkIfSameReplicaIsPrimary(looper: Looper, replicas: Sequence[TestReplica] = None, retryWait: float = 1, timeout: float = 20): # One and only one primary should be found and every replica should agree # on same primary def checkElectionDone(): unknowns = [r for r in replicas if r.primaryName is None] assert len(unknowns) == 0, "election should be complete, " \ "but {} out of {} ({}) don't know who the primary " \ "is for protocol instance {}". \ format(len(unknowns), len(replicas), unknowns, replicas[0].instId) def checkPrisAreOne(): # number of expected primaries pris = sum(1 for r in replicas if r.isPrimary) assert pris == 1, "Primary count should be 1, but was {} for " \ "protocol no {}".format(pris, replicas[0].instId) def checkPrisAreSame(): pris = {r.primaryName for r in replicas} assert len(pris) == 1, "Primary should be same for all, but were {} " \ "for protocol no {}" \ .format(pris, replicas[0].instId) looper.run( eventuallyAll(checkElectionDone, checkPrisAreOne, checkPrisAreSame, retryWait=retryWait, totalTimeout=timeout))
def checkPropagated(looper, nodeSet, request, faultyNodes=0, timeout=10): nodesSize = len(list(nodeSet.nodes)) # noinspection PyIncorrectDocstring def g(node: TestNode): """ 1. no of propagate received by node must be n -1 with zero faulty nodes in system; where n = num of nodes 2. no of propagate received by node must be greater than or equal to f + 1 """ actualMsgs = len([ x for x in getAllArgs(node, Node.processPropagate) if x['msg'].request[f.REQ_ID.nm] == request.reqId and x['msg'].request[f.IDENTIFIER.nm] == request.identifier and x['msg'].request[OPERATION] == request.operation ]) numOfMsgsWithZFN = nodesSize - 1 numOfMsgsWithFaults = faultyNodes + 1 assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) coros = [partial(g, node) for node in nodeSet] looper.run( eventuallyAll(*coros, totalTimeout=timeout, acceptableFails=faultyNodes))
def checkIfSameReplicaIsPrimary(looper: Looper, replicas: Sequence[TestReplica] = None, retryWait: float = 1, timeout: float = 20): # One and only one primary should be found and every replica should agree # on same primary def checkElectionDone(): unknowns = [r for r in replicas if r.primaryName is None] assert len(unknowns) == 0, "election should be complete, " \ "but {} out of {} ({}) don't know who the primary " \ "is for protocol instance {}". \ format(len(unknowns), len(replicas), unknowns, replicas[0].instId) def checkPrisAreOne(): # number of expected primaries pris = sum(1 for r in replicas if r.isPrimary) assert pris == 1, "Primary count should be 1, but was {} for " \ "protocol no {}".format(pris, replicas[0].instId) def checkPrisAreSame(): pris = {r.primaryName for r in replicas} assert len(pris) == 1, "Primary should be same for all, but were {} " \ "for protocol no {}" \ .format(pris, replicas[0].instId) looper.run( eventuallyAll(checkElectionDone, checkPrisAreOne, checkPrisAreSame, retryWait=retryWait, totalTimeout=timeout))
def wait_for_requests_ordered(looper, nodes, requests): node_count = len(nodes) timeout_per_request = waits.expectedTransactionExecutionTime(node_count) total_timeout = (1 + len(requests) / 10) * timeout_per_request coros = [partial(check_request_ordered, node, request) for (node, request) in list(itertools.product(nodes, requests))] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=total_timeout))
def check_reqacks(client, looper, reqs, txnPoolNodeSet): reqack_coros = [] for req in reqs: reqack_coros.extend([ partial(checkReqAck, client, node, req.identifier, req.reqId, None) for node in txnPoolNodeSet ]) timeout = waits.expectedReqAckQuorumTime() looper.run(eventuallyAll(*reqack_coros, totalTimeout=timeout))
def wait_for_requests_ordered(looper, nodes, requests): node_count = len(nodes) timeout_per_request = waits.expectedTransactionExecutionTime(node_count) total_timeout = (1 + len(requests) / 10) * timeout_per_request coros = [ partial(check_request_ordered, node, request) for (node, request) in list(itertools.product(nodes, requests)) ] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=total_timeout))
def sdk_check_request_is_not_returned_to_nodes(looper, nodeSet, request): instances = range(getNoInstances(len(nodeSet))) coros = [] for node, inst_id in itertools.product(nodeSet, instances): c = partial(checkRequestNotReturnedToNode, node=node, identifier=request['identifier'], reqId=request['reqId'], instId=inst_id) coros.append(c) timeout = waits.expectedTransactionExecutionTime(len(nodeSet)) looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def sdk_check_request_is_not_returned_to_nodes(looper, nodeSet, request): instances = range(getNoInstances(len(nodeSet))) coros = [] for node, inst_id in itertools.product(nodeSet, instances): c = partial(checkRequestNotReturnedToNode, node=node, identifier=request['identifier'], reqId=request['reqId'], instId=inst_id ) coros.append(c) timeout = waits.expectedTransactionExecutionTime(len(nodeSet)) looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def testRequestFullRoundTrip(restrictiveVerifier, client1, sent1, looper, txnPoolNodeSet): update = {'reason': 'client request invalid: InvalidClientRequest() ' '[caused by amount too high\nassert 999 <= 100]'} coros2 = [partial(checkReqNack, client1, node, sent1.identifier, sent1.reqId, update) for node in txnPoolNodeSet] timeout = waits.expectedReqAckQuorumTime() looper.run(eventuallyAll(*coros2, totalTimeout=timeout))
def checkSufficientRepliesForRequests(looper, client, requests, fVal=None, timeoutPerReq=None): nodeCount = len(client.nodeReg) fVal = fVal or getMaxFailures(nodeCount) timeoutPerReq = timeoutPerReq or 5 * nodeCount coros = [] for request in requests: coros.append( partial(checkSufficientRepliesRecvd, client.inBox, request.reqId, fVal)) looper.run( eventuallyAll(*coros, retryWait=1, totalTimeout=timeoutPerReq * len(requests)))
def testClientRetryRequestWhenReplyNotReceived(looper, txnPoolNodeSet, client1, wallet1, tconf): """ A node say Alpha sends ACK but doesn't send REPLY. The client resends the request and gets REPLY """ alpha = txnPoolNodeSet[0] skipped = False origTrans = alpha.transmitToClient def skipReplyOnce(msg, remoteName): nonlocal skipped if isinstance(msg, Reply) and not skipped: skipped = True return origTrans(msg, remoteName) alpha.transmitToClient = skipReplyOnce req = sendRandomRequest(wallet1, client1) coros = [ partial(checkReqAck, client1, node, *req.key) for node in txnPoolNodeSet ] timeout = waits.expectedReqAckQuorumTime() start = time.perf_counter() looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=timeout)) idr, reqId = req.key # Client should get only 3 replies till the retry timeout since one node # is not sending any replies wait_for_replies(looper, client1, idr, reqId, 3, custom_timeout=tconf.CLIENT_REPLY_TIMEOUT - 1) end = time.perf_counter() # Client should wait till the retry timeout but after that should # get the reply from the remaining node looper.runFor(tconf.CLIENT_REPLY_TIMEOUT - (end - start)) wait_for_replies(looper, client1, idr, reqId, 4)
def wait_for_elections_done_on_given_nodes(looper: Looper, nodes: Iterable[Node], num_of_instances: int, timeout: float, retry_wait: float=1.0): """ Wait for primary elections to be completed on all the replicas of the given nodes. """ def check_num_of_replicas(): for node in nodes: assert len(node.replicas) == num_of_instances def verify_each_replica_knows_its_primary(): for node in nodes: for inst_id, replica in node.replicas.items(): assert replica.hasPrimary looper.run(eventuallyAll(check_num_of_replicas, verify_each_replica_knows_its_primary, totalTimeout=timeout, retryWait=retry_wait))
def wait_for_elections_done_on_given_nodes(looper: Looper, nodes: Iterable[Node], num_of_instances: int, timeout: float, retry_wait: float=1.0): """ Wait for primary elections to be completed on all the replicas of the given nodes. """ def check_num_of_replicas(): for node in nodes: assert len(node.replicas) == num_of_instances def verify_each_replica_knows_its_primary(): for node in nodes: for inst_id, replica in node.replicas.items(): assert replica.hasPrimary looper.run(eventuallyAll(check_num_of_replicas, verify_each_replica_knows_its_primary, totalTimeout=timeout, retryWait=retry_wait))
def testClientRetryRequestWhenReplyNotReceived(looper, nodeSet, client1, wallet1, tconf): """ A node say Alpha sends ACK but doesn't send REPLY. The connect resends the request and gets REPLY """ alpha = nodeSet.Alpha skipped = False origTrans = alpha.transmitToClient def skipReplyOnce(msg, remoteName): nonlocal skipped if isinstance(msg, Reply) and not skipped: skipped = True return origTrans(msg, remoteName) alpha.transmitToClient = skipReplyOnce req = sendRandomRequest(wallet1, client1) coros = [partial(checkReqAck, client1, node, *req.key) for node in nodeSet] looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=3)) looper.run( eventually(checkReplyCount, client1, *req.key, 3, retryWait=1, timeout=3)) looper.run( eventually(checkReplyCount, client1, *req.key, 4, retryWait=1, timeout=tconf.CLIENT_REPLY_TIMEOUT + 5))
def checkPrepared(looper, nodeSet, preprepared1, instIds, faultyNodes=0, timeout=30): nodeCount = len(list(nodeSet.nodes)) f = getMaxFailures(nodeCount) def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primaryDontSendAnyPREPAREs(): """ 1. no of PREPARE sent by primary should be 0 """ for r in allReplicas: for param in getAllArgs(r, Replica.processPrepare): sender = param['sender'] assert sender != primary.name def allReplicasSeeCorrectNumberOfPREPAREs(): """ 1. no of PREPARE received by replicas must be n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f for replica in allReplicas: key = primary.viewNo, primary.lastPrePrepareSeqNo if key in replica.prepares: actualMsgs = len(replica.prepares[key].voters) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(allReplicas) - faultyNodes def primaryReceivesCorrectNumberOfPREPAREs(): """ num of PREPARE seen by primary replica is n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ actualMsgs = len([ param for param in getAllArgs(primary, primary.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.lastPrePrepareSeqNo) and param['sender'] != primary.name ]) numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f - 1 assert msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) # TODO what if the primary is faulty? def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): """ num of PREPARE seen by Non primary replica is n - 2 without faults and 2f - 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 2 numOfMsgsWithFaults = (2 * f) - 1 for npr in nonPrimaryReplicas: actualMsgs = len([ param for param in getAllArgs(npr, npr.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == ( primary.instId, primary.viewNo, primary.lastPrePrepareSeqNo) ]) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes # TODO how do we know if one of the faulty nodes is a primary or # not? primaryDontSendAnyPREPAREs() allReplicasSeeCorrectNumberOfPREPAREs() primaryReceivesCorrectNumberOfPREPAREs() nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs() coros = [partial(g, instId) for instId in instIds] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def testBankReqValidationPlugin(looper, nodeSet, client1, wallet1, tdir, pluginVerPath): plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" client2, wallet2 = setupClient(looper, nodeSet, tmpdir=tdir) req = submitOp(wallet1, client1, { TXN_TYPE: "dummy", TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 } }) validTypes = ', '.join(plugin.validTxnTypes) update = { 'reason': makeReason( commonError, "dummy is not a valid " "transaction type, must be " "one of {}".format(validTypes)) } coros1 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, }) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } coros2 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: "some string" }) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } coros3 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: -3 } }) update = { 'reason': makeReason( commonError, "{} must be present and should be " "a number greater than 0".format(AMOUNT)) } coros4 = [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] timeout = waits.expectedReqAckQuorumTime() looper.run( eventuallyAll(*(coros1 + coros2 + coros3 + coros4), totalTimeout=timeout)) req = submitOp(wallet1, client1, { TXN_TYPE: CREDIT, TARGET_NYM: wallet2.defaultId, DATA: { AMOUNT: 30 } }) waitForSufficientRepliesForRequests(looper, client1, requests=[req], fVal=1) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 1
def testAuctionReqValidationPlugin(looper, nodeSet, wallet1, client1, tdir, pluginVerPath): # TODO: Test more cases plugin = PluginLoader(pluginVerPath) plugin = next(iter(plugin.plugins[PLUGIN_TYPE_VERIFICATION])) commonError = "client request invalid: InvalidClientRequest()" allCoros = [] op = {TXN_TYPE: "dummy", DATA: {AMOUNT: 30}} req = submitOp(wallet1, client1, op) validTypes = ', '.join(plugin.validTxnTypes) update = { 'reason': makeReason( commonError, "dummy is not a valid transaction " "type, must be one of {}".format(validTypes)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = { TXN_TYPE: AUCTION_START, } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = { TXN_TYPE: PLACE_BID, } req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: PLACE_BID, DATA: "some string"} req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} attribute is missing or not in proper format".format(DATA)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: PLACE_BID, DATA: {AMOUNT: 453}} req = submitOp(wallet1, client1, op) update = {'reason': makeReason(commonError, "No id provided for auction")} allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: AUCTION_START, DATA: {}} req = submitOp(wallet1, client1, op) update = {'reason': makeReason(commonError, "No id provided for auction")} allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] op = {TXN_TYPE: AUCTION_END, DATA: {}} req = submitOp(wallet1, client1, op) update = {'reason': makeReason(commonError, "No id provided for auction")} allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] auctionId = str(uuid4()) op = {TXN_TYPE: PLACE_BID, DATA: {ID: auctionId, AMOUNT: -3}} req = submitOp(wallet1, client1, op) update = { 'reason': makeReason( commonError, "{} must be present and should be " "a number greater than 0".format(AMOUNT)) } allCoros += [ partial(checkReqNack, client1, node, req.identifier, req.reqId, update) for node in nodeSet ] timeout = waits.expectedReqAckQuorumTime() looper.run(eventuallyAll(*allCoros, totalTimeout=timeout)) for n in nodeSet: # type: Node opVerifier, = n.opVerifiers assert opVerifier.count == 0
def checkPrePrepared(looper, nodeSet, propagated1, instIds, faultyNodes=0, timeout=30): nodesSize = len(list(nodeSet)) def g(instId): primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primarySeesCorrectNumberOfPREPREPAREs(): """ no of PRE-PREPARE as seen by processPrePrepare method for primary must be 0 with or without faults in system """ l1 = len([ param for param in getAllArgs(primary, primary.processPrePrepare) ]) assert l1 == 0 def nonPrimarySeesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be 1; whn zero faulty nodes in system. 2. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be greater than or equal to 0; with faults in system. """ expectedPrePrepareRequest = PrePrepare(instId, primary.viewNo, primary.lastPrePrepareSeqNo, propagated1.identifier, propagated1.reqId, propagated1.digest, time.time()) passes = 0 for npr in nonPrimaryReplicas: actualMsgs = len([ param for param in getAllArgs(npr, npr.processPrePrepare) if (param['pp'][:-1], param['sender']) == (expectedPrePrepareRequest[:-1], primary.name) ]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += int( msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes def primarySentsCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE sent by primary is 1 with or without fault in system but, when primary is faulty no of sent PRE_PREPARE will be zero and primary must be marked as malicious. """ actualMsgs = len([ param for param in getAllArgs(primary, primary.doPrePrepare) if (param['reqDigest'].identifier, param['reqDigest'].reqId, param['reqDigest'].digest) == (propagated1.identifier, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 # TODO: Considering, Primary is not faulty and will always send # PRE-PREPARE. Write separate test for testing when Primary # is faulty assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithZFN) def nonPrimaryReceivesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE received by non-primaries must be 1 with zero faults in system, and 0 faults in system. """ passes = 0 for npr in nonPrimaryReplicas: l4 = len([ param for param in getAllArgs(npr, npr.addToPrePrepares) if (param['pp'].identifier, param['pp'].reqId, param['pp'].digest) == (propagated1.identifier, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += msgCountOK(nodesSize, faultyNodes, l4, numOfMsgsWithZFN, numOfMsgsWithFaults) assert passes >= len(nonPrimaryReplicas) - faultyNodes primarySeesCorrectNumberOfPREPREPAREs() nonPrimarySeesCorrectNumberOfPREPREPAREs() primarySentsCorrectNumberOfPREPREPAREs() nonPrimaryReceivesCorrectNumberOfPREPREPAREs() coros = [partial(g, instId) for instId in instIds] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def checkCommitted(looper, nodeSet, prepared1, instIds, faultyNodes=0, timeout=60): nodeCount = len((list(nodeSet))) f = getMaxFailures(nodeCount) def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primaryReplica = getPrimaryReplica(nodeSet, instId) def replicasSeesCorrectNumOfCOMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to 2f + 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount numOfMsgsWithFault = (2 * f) + 1 key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] actualMsgsReceived = len(rcvdCommitRqst.voters) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= len(allReplicas) - faultyNodes def replicasReceivesCorrectNumberOfCOMMITs(): """ num of commit messages seen by replica must be equal to n - 1; when zero fault and greater than or equal to 2f+1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFault = 2 * f for r in allReplicas: args = getAllArgs(r, r.processCommit) actualMsgsReceived = len(args) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) for arg in args: assert arg['commit'].viewNo == primaryReplica.viewNo and \ arg['commit'].ppSeqNo == primaryReplica.lastPrePrepareSeqNo and \ arg['commit'].digest == prepared1.digest assert r.name != arg['sender'] assert passes >= len(allReplicas) - faultyNodes replicasReceivesCorrectNumberOfCOMMITs() replicasSeesCorrectNumOfCOMMITs() coros = [partial(g, instId) for instId in instIds] looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))