def test_belated_request_not_processed_if_already_in_3pc_process( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(300)) one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1) sdk_send_signed_requests(sdk_pool_handle, one_req) looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) delta.clientIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) for node in txnPoolNodeSet: node.nodeIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedOrderingTime(delta.replicas.num_replicas)) for node in txnPoolNodeSet: assert node.domainLedger.size - initial_ledger_size == 1
def test_belated_request_not_processed_if_already_in_3pc_process( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(300)) one_req = sdk_signed_random_requests(looper, sdk_wallet_client, 1) sdk_send_signed_requests(sdk_pool_handle, one_req) looper.runFor( waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) delta.clientIbStasher.reset_delays_and_process_delayeds() looper.runFor( waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) for node in txnPoolNodeSet: node.nodeIbStasher.reset_delays_and_process_delayeds() looper.runFor(waits.expectedOrderingTime(delta.replicas.num_replicas)) for node in txnPoolNodeSet: assert node.domainLedger.size - initial_ledger_size == 1
def test_unordered_request_freed_on_replica_removal(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, chkFreqPatched, view_change): node = txnPoolNodeSet[0] # Stabilize checkpoint # Send one more request to stabilize checkpoint sdk_send_random_and_check( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, CHK_FREQ - get_pp_seq_no(txnPoolNodeSet) % CHK_FREQ) old_stable_checkpoint = node.master_replica._consensus_data.stable_checkpoint stashers = [n.nodeIbStasher for n in txnPoolNodeSet] with delay_rules(stashers, cDelay(delay=sys.maxsize), msg_rep_delay(types_to_delay=[COMMIT])): req = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) looper.runFor( waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) f_d, f_r = get_forwarded_to_all(node) assert f_d node.replicas.remove_replica(node.replicas.num_replicas - 1) assert node.requests[f_d].forwardedTo == node.replicas.num_replicas check_for_nodes(txnPoolNodeSet, check_stable_checkpoint, old_stable_checkpoint) sdk_get_replies(looper, req) check_for_nodes(txnPoolNodeSet, check_stable_checkpoint, old_stable_checkpoint) # Send one more request to stabilize checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, CHK_FREQ - 1) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, old_stable_checkpoint + CHK_FREQ)) assert len(node.requests) == 0
def checkCommitted(looper, txnPoolNodeSet, prepared1, instIds, faultyNodes=0): timeout = waits.expectedCommittedTime(len(txnPoolNodeSet)) nodeCount = len((list(txnPoolNodeSet))) quorums = Quorums(nodeCount) def g(instId): allReplicas = getAllReplicas(txnPoolNodeSet, instId) primaryReplica = getPrimaryReplica(txnPoolNodeSet, instId) def replicas_gets_correct_num_of_COMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to n-f with faults. """ passes = 0 numOfMsgsWithZFN = quorums.commit.value numOfMsgsWithFault = quorums.commit.value key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo) for r in allReplicas: if key in r._ordering_service.commits: rcvdCommitRqst = r._ordering_service.commits[key] actualMsgsReceived = len(rcvdCommitRqst.voters) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= min(len(allReplicas) - faultyNodes, numOfMsgsWithZFN) replicas_gets_correct_num_of_COMMITs() funcs = [partial(g, instId) for instId in instIds] # TODO Select or create the timeout from 'waits'. Don't use constant. # looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout)) chk_all_funcs(looper, funcs, faultyNodes, timeout)
def checkCommitted(looper, txnPoolNodeSet, prepared1, instIds, faultyNodes=0): timeout = waits.expectedCommittedTime(len(txnPoolNodeSet)) nodeCount = len((list(txnPoolNodeSet))) quorums = Quorums(nodeCount) def g(instId): allReplicas = getAllReplicas(txnPoolNodeSet, instId) primaryReplica = getPrimaryReplica(txnPoolNodeSet, instId) def replicas_gets_correct_num_of_COMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to n-f with faults. """ passes = 0 numOfMsgsWithZFN = quorums.commit.value numOfMsgsWithFault = quorums.commit.value key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] actualMsgsReceived = len(rcvdCommitRqst.voters) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= min(len(allReplicas) - faultyNodes, numOfMsgsWithZFN) replicas_gets_correct_num_of_COMMITs() funcs = [partial(g, instId) for instId in instIds] # TODO Select or create the timeout from 'waits'. Don't use constant. # looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout)) chk_all_funcs(looper, funcs, faultyNodes, timeout)
def test_unordered_request_freed_on_replica_removal(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, chkFreqPatched, view_change): node = txnPoolNodeSet[0] stashers = [n.nodeIbStasher for n in txnPoolNodeSet] with delay_rules(stashers, cDelay(delay=sys.maxsize)): req = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) assert len(node.requests) == 1 forwardedToBefore = next(iter(node.requests.values())).forwardedTo node.replicas.remove_replica(node.replicas.num_replicas - 1) assert len(node.requests) == 1 forwardedToAfter = next(iter(node.requests.values())).forwardedTo assert forwardedToAfter == forwardedToBefore - 1 chkChkpoints(txnPoolNodeSet, 0) sdk_get_replies(looper, req) chkChkpoints(txnPoolNodeSet, 1) # Send one more request to stabilize checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0)) assert len(node.requests) == 0
def test_unordered_request_freed_on_replica_removal(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, chkFreqPatched, view_change): node = txnPoolNodeSet[0] stashers = [n.nodeIbStasher for n in txnPoolNodeSet] with delay_rules(stashers, cDelay(delay=sys.maxsize)): req = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) + waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + waits.expectedPrepareTime(len(txnPoolNodeSet)) + waits.expectedCommittedTime(len(txnPoolNodeSet))) assert len(node.requests) == 1 forwardedToBefore = next(iter(node.requests.values())).forwardedTo node.replicas.remove_replica(node.replicas.num_replicas - 1) assert len(node.requests) == 1 forwardedToAfter = next(iter(node.requests.values())).forwardedTo assert forwardedToAfter == forwardedToBefore - 1 chkChkpoints(txnPoolNodeSet, 0) sdk_get_replies(looper, req) chkChkpoints(txnPoolNodeSet, 1) # Send one more request to stabilize checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0)) assert len(node.requests) == 0