def committed1(looper, nodeSet, client1, prepared1, faultyNodes): checkCommited(looper, nodeSet, prepared1, range(getNoInstances(len(nodeSet))), faultyNodes) return prepared1
def testPrimarySelectionAfterViewChange( # noqa looper, txnPoolNodeSet, primaryReplicas, catchup_complete_count): """ Test that primary replica of a protocol instance shifts to a new node after a view change. """ # TODO: This test can fail due to view change. ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) for n in txnPoolNodeSet: assert n.spylog.count( n.allLedgersCaughtUp) > catchup_complete_count[n.name] # Primary replicas before view change prBeforeVC = primaryReplicas # Primary replicas after view change instanceCount = getNoInstances(nodeCount) prAfterVC = [getPrimaryReplica(txnPoolNodeSet, i) for i in range(instanceCount)] # Primary replicas have moved to the next node for br, ar in zip(prBeforeVC, prAfterVC): assert ar.node.rank - br.node.rank == 1 check_rank_consistent_across_each_node(txnPoolNodeSet) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1)
def testPrimarySelectionAfterViewChange( # noqa looper, txnPoolNodeSet, primaryReplicas, catchup_complete_count, view_change_done): """ Test that primary replica of a protocol instance shifts to a new node after a view change. """ # TODO: This test can fail due to view change. for n in txnPoolNodeSet: assert n.spylog.count( n.allLedgersCaughtUp) > catchup_complete_count[n.name] # Primary replicas before view change prBeforeVC = primaryReplicas # Primary replicas after view change instanceCount = getNoInstances(nodeCount) prAfterVC = [ getPrimaryReplica(txnPoolNodeSet, i) for i in range(instanceCount) ] # Primary replicas have moved to the next node for br, ar in zip(prBeforeVC, prAfterVC): assert ar.node.rank - br.node.rank == 1 check_rank_consistent_across_each_node(txnPoolNodeSet) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1)
def preprepared1(looper, nodeSet, propagated1, faultyNodes): checkPrePrepared(looper, nodeSet, propagated1, range(getNoInstances(len(nodeSet))), faultyNodes) return propagated1
def committed1(looper, txnPoolNodeSet, prepared1, faultyNodes): checkCommitted(looper, txnPoolNodeSet, prepared1, range(getNoInstances(len(txnPoolNodeSet))), faultyNodes) return prepared1
def preprepared1(looper, txnPoolNodeSet, propagated1, faultyNodes): checkPrePrepared(looper, txnPoolNodeSet, propagated1, range(getNoInstances(len(txnPoolNodeSet))), faultyNodes) return propagated1
def test_num_of_commit_msg_with_2_of_6_faulty(evilNodes, looper, txnPoolNodeSet, prepared1, noRetryReq): with pytest.raises(AssertionError): # To raise an error pass less than the actual number of faults checkCommitted(looper, txnPoolNodeSet, prepared1, range(getNoInstances(len(txnPoolNodeSet))), f)
def committed1(looper, nodeSet, client1, prepared1, faultyNodes): checkCommitted(looper, nodeSet, prepared1, range(getNoInstances(len(nodeSet))), faultyNodes) return prepared1
def check_request_is_not_returned_to_nodes(txnPoolNodeSet, request): instances = range(getNoInstances(len(txnPoolNodeSet))) for node, inst_id in itertools.product(txnPoolNodeSet, instances): checkRequestNotReturnedToNode(node, request.identifier, request.reqId, inst_id)
def preprepared1WithDelay(looper, txnPoolNodeSet, propagated1, faultyNodes): timeouts = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + delayPrePrepareSec checkPrePrepared(looper, txnPoolNodeSet, propagated1, range(getNoInstances(len(txnPoolNodeSet))), faultyNodes, timeout=timeouts)
def testNumOfCommitMsgsWithFPlusOneFaults(afterElection, looper, nodeSet, prepared1, noRetryReq): with pytest.raises(AssertionError): checkCommited(looper, nodeSet, prepared1, range(getNoInstances(len(nodeSet))), faultyNodes)
def testNumOfCommitMsgsWithFPlusOneFaults(afterElection, looper, txnPoolNodeSet, prepared1, noRetryReq): with pytest.raises(AssertionError): # To raise an error pass less than the actual number of faults checkCommitted(looper, txnPoolNodeSet, prepared1, range(getNoInstances(len(txnPoolNodeSet))), faultyNodes - 1)
def checkOrderedCount(): instances = getNoInstances(len(nodeSet)) resp = [ requestReturnedToNode(node, wallet1.defaultId, committed1.reqId, instId) for node in nodeSet for instId in range(instances) ] assert resp.count(True) >= (len(nodeSet) - faultyNodes) * instances
def testNumOfCommitMsgsWithFPlusOneFaults(afterElection, looper, nodeSet, prepared1, noRetryReq): with pytest.raises(AssertionError): checkCommitted(looper, nodeSet, prepared1, range(getNoInstances(len(nodeSet))), faultyNodes)
def test_num_of_prepare_2_of_6_faulty(evilNodes, looper, txnPoolNodeSet, preprepared1, noRetryReq): with pytest.raises(AssertionError): # To raise an error pass less than the actual number of faults checkPrepared(looper, txnPoolNodeSet, preprepared1, range(getNoInstances(len(txnPoolNodeSet))), f)
def do_view_change_with_propagate_primary_on_one_delayed_node( slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client): slow_stasher = slow_node.nodeIbStasher fast_nodes = [n for n in nodes if n != slow_node] stashers = [n.nodeIbStasher for n in nodes] # Get last prepared certificate in pool lpc = last_prepared_certificate(nodes) # Get pool current view no view_no = lpc[0] with delay_rules(slow_stasher, icDelay()): with delay_rules(slow_stasher, vcd_delay()): with delay_rules(stashers, cDelay()): # Send request request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) # Wait until this request is prepared on N-f nodes looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1))) # Trigger view change for n in nodes: n.view_changer.on_master_degradation() # Wait until view change is completed on all nodes except slow one waitForViewChange(looper, fast_nodes, expectedViewNo=view_no + 1, customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes))) wait_for_elections_done_on_given_nodes(looper, fast_nodes, getRequiredInstances(len(nodes)), timeout=waits.expectedPoolElectionTimeout(len(nodes))) # Now all the nodes receive Commits # The slow node will accept Commits and order the 3PC-batch in the old view looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes)))) # Now slow node receives ViewChangeDones waitForViewChange(looper, [slow_node], expectedViewNo=view_no + 1, customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes))) wait_for_elections_done_on_given_nodes(looper, [slow_node], getRequiredInstances(len(nodes)), timeout=waits.expectedPoolElectionTimeout(len(nodes))) # Now slow node receives InstanceChanges but discards them because already # started propagate primary to the same view. # Finish request gracefully sdk_get_reply(looper, request)
def test_cannot_restore_last_sent_pp_seq_no_if_replica_absent( txnPoolNodeSet, view_no_set, setup): node = txnPoolNodeSet[0] assert node.viewNo == 2 absent_replica_index = getNoInstances(nodeCount) can = node.last_sent_pp_store_helper._can_restore_last_sent_pp_seq_no( absent_replica_index, [2, 5]) assert can is False
def test_cannot_restore_last_sent_pp_seq_no_if_replica_absent( txnPoolNodeSet, view_no_set, setup): node = txnPoolNodeSet[0] assert node.viewNo == 2 absent_replica_index = getNoInstances(nodeCount) can = node.last_sent_pp_store_helper._can_restore_last_sent_pp_seq_no( PrePrepareKey(inst_id=absent_replica_index, view_no=2, pp_seq_no=5)) assert can is False
def sdk_check_request_is_not_returned_to_nodes(looper, nodeSet, request): instances = range(getNoInstances(len(nodeSet))) coros = [] for node, inst_id in itertools.product(nodeSet, instances): c = partial(checkRequestNotReturnedToNode, node=node, identifier=request['identifier'], reqId=request['reqId'], instId=inst_id) coros.append(c) timeout = waits.expectedTransactionExecutionTime(len(nodeSet)) looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def sdk_check_request_is_not_returned_to_nodes(looper, nodeSet, request): instances = range(getNoInstances(len(nodeSet))) coros = [] for node, inst_id in itertools.product(nodeSet, instances): c = partial(checkRequestNotReturnedToNode, node=node, identifier=request['identifier'], reqId=request['reqId'], instId=inst_id ) coros.append(c) timeout = waits.expectedTransactionExecutionTime(len(nodeSet)) looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas, viewChangeDone): """ Test that primary replica of a protocol instance shifts to a new node after a view change. """ # Primary replicas before view change prBeforeVC = primaryReplicas # Primary replicas after view change instanceCount = getNoInstances(nodeCount) prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)] # Primary replicas have moved to the next node for br, ar in zip(prBeforeVC, prAfterVC): assert ar.node.rank - br.node.rank == 1 checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def replied1(looper, nodeSet, client1, committed1, wallet1, faultyNodes): numOfNodes = len(nodeSet) numOfInstances = getNoInstances(numOfNodes) quorum = numOfInstances * (numOfNodes - faultyNodes) def checkOrderedCount(): resp = [ requestReturnedToNode(node, wallet1.defaultId, committed1.reqId, instId) for node in nodeSet for instId in range(numOfInstances) ] assert resp.count(True) >= quorum orderingTimeout = waits.expectedOrderingTime(numOfInstances) looper.run( eventually(checkOrderedCount, retryWait=1, timeout=orderingTimeout)) waitForSufficientRepliesForRequests(looper, client1, requests=[committed1]) return committed1
def replied1(looper, nodeSet, client1, committed1): for instId in range(getNoInstances(len(nodeSet))): getPrimaryReplica(nodeSet, instId) looper.run(*[eventually(checkRequestReturnedToNode, node, client1.defaultIdentifier, committed1.reqId, committed1.digest, instId, retryWait=1, timeout=30) for node in nodeSet]) looper.run(eventually( checkSufficientRepliesRecvd, client1.inBox, committed1.reqId, 2, retryWait=2, timeout=30)) return committed1
def replied1(looper, txnPoolNodeSet, sdk_wallet_client, committed1, faultyNodes, sent1): numOfNodes = len(txnPoolNodeSet) numOfInstances = getNoInstances(numOfNodes) quorum = numOfInstances * (numOfNodes - faultyNodes) _, did = sdk_wallet_client def checkOrderedCount(): resp = [ requestReturnedToNode(node, committed1.digest, instId) for node in txnPoolNodeSet for instId in range(numOfInstances) ] assert resp.count(True) >= quorum orderingTimeout = waits.expectedOrderingTime(numOfInstances) looper.run( eventually(checkOrderedCount, retryWait=1, timeout=orderingTimeout)) sdk_get_and_check_replies(looper, sent1) return committed1
def replied1(looper, txnPoolNodeSet, sdk_wallet_client, committed1, faultyNodes, sent1): numOfNodes = len(txnPoolNodeSet) numOfInstances = getNoInstances(numOfNodes) quorum = numOfInstances * (numOfNodes - faultyNodes) _, did = sdk_wallet_client def checkOrderedCount(): resp = [requestReturnedToNode(node, committed1.digest, instId) for node in txnPoolNodeSet for instId in range(numOfInstances)] assert resp.count(True) >= quorum orderingTimeout = waits.expectedOrderingTime(numOfInstances) looper.run(eventually(checkOrderedCount, retryWait=1, timeout=orderingTimeout)) sdk_get_and_check_replies(looper, sent1) return committed1
def replied1(looper, nodeSet, client1, committed1, wallet1): for instId in range(getNoInstances(len(nodeSet))): getPrimaryReplica(nodeSet, instId) looper.run(*[ eventually(checkRequestReturnedToNode, node, wallet1.defaultId, committed1.reqId, committed1.digest, instId, retryWait=1, timeout=30) for node in nodeSet ]) looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, committed1.reqId, 2, retryWait=2, timeout=30)) return committed1
def primaryReplicas(nodeSet, up): instanceCount = getNoInstances(nodeCount) return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
def primaryReplicas(txnPoolNodeSet): instanceCount = getNoInstances(nodeCount) return [getPrimaryReplica(txnPoolNodeSet, i) for i in range(instanceCount)]
def checkOrderedCount(): instances = getNoInstances(len(nodeSet)) resp = [requestReturnedToNode(node, wallet1.defaultId, committed1.reqId, instId) for node in nodeSet for instId in range(instances)] assert resp.count(True) >= (len(nodeSet) - faultyNodes)*instances
def check_request_is_not_returned_to_nodes(txnPoolNodeSet, request): instances = range(getNoInstances(len(txnPoolNodeSet))) for node, inst_id in itertools.product(txnPoolNodeSet, instances): checkRequestNotReturnedToNode(node, request.key, inst_id)