def test_commits_recvd_first(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): slow_node = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)][-1] other_nodes = [n for n in txnPoolNodeSet if n != slow_node] delay = 50 slow_node.nodeIbStasher.delay(ppDelay(delay, 0)) slow_node.nodeIbStasher.delay(pDelay(delay, 0)) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=20, num_batches=4) assert not slow_node.master_replica.prePrepares assert not slow_node.master_replica.prepares assert not slow_node.master_replica.commits assert len(slow_node.master_replica.commitsWaitingForPrepare) > 0 slow_node.reset_delays_and_process_delayeds() waitNodeDataEquality(looper, slow_node, *other_nodes) assert check_if_all_equal_in_list( [n.master_replica.ordered for n in txnPoolNodeSet]) assert slow_node.master_replica.prePrepares assert slow_node.master_replica.prepares assert slow_node.master_replica.commits assert not slow_node.master_replica.commitsWaitingForPrepare
def chk_if_equal_txn_to_3pc(nodes, count=None): txn_to_tpc = [] for node in nodes: txn_to_tpc.append(node.txn_seq_range_to_3phase_key[DOMAIN_LEDGER_ID]) assert check_if_all_equal_in_list(txn_to_tpc) if count is not None: assert len(txn_to_tpc[0]) == count
def test_dequeue_and_validate_commits(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): slow_node = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)][-1] other_nodes = [n for n in txnPoolNodeSet if n != slow_node] delay = 50 with delay_rules(slow_node.nodeIbStasher, pDelay(delay), msg_rep_delay(delay, [PREPARE, PREPREPARE])): with delay_rules(slow_node.nodeIbStasher, ppDelay(delay)): sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=1, num_batches=1) assert not slow_node.master_replica._ordering_service.prePrepares assert not slow_node.master_replica._ordering_service.prepares assert not slow_node.master_replica._ordering_service.commits assert len(slow_node.master_replica._ordering_service. commitsWaitingForPrepare) > 0 waitNodeDataEquality(looper, slow_node, *other_nodes) assert check_if_all_equal_in_list([ n.master_replica._ordering_service.ordered for n in txnPoolNodeSet ]) assert slow_node.master_replica._ordering_service.prePrepares assert slow_node.master_replica._ordering_service.prepares assert slow_node.master_replica._ordering_service.commits assert not slow_node.master_replica._ordering_service.commitsWaitingForPrepare assert all(slow_node.master_replica.last_ordered_3pc == n.master_replica.last_ordered_3pc for n in other_nodes)
def last_prepared(nodes): lst = [ n.master_replica.last_prepared_certificate_in_view() for n in nodes ] # All nodes have same last prepared assert check_if_all_equal_in_list(lst) return lst[0]
def test_commits_recvd_first(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): slow_node = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)][-1] other_nodes = [n for n in txnPoolNodeSet if n != slow_node] delay = 50 slow_node.nodeIbStasher.delay(ppDelay(delay, 0)) slow_node.nodeIbStasher.delay(pDelay(delay, 0)) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=20, num_batches=4) assert not slow_node.master_replica.prePrepares assert not slow_node.master_replica.prepares assert not slow_node.master_replica.commits assert len(slow_node.master_replica.commitsWaitingForPrepare) > 0 slow_node.reset_delays_and_process_delayeds() waitNodeDataEquality(looper, slow_node, *other_nodes) assert check_if_all_equal_in_list([n.master_replica.ordered for n in txnPoolNodeSet]) assert slow_node.master_replica.prePrepares assert slow_node.master_replica.prepares assert slow_node.master_replica.commits assert not slow_node.master_replica.commitsWaitingForPrepare
def chk_if_equal_txn_to_3pc(nodes, count=None): txn_to_tpc = [] for node in nodes: txn_to_tpc.append(node.txn_seq_range_to_3phase_key[DOMAIN_LEDGER_ID]) assert check_if_all_equal_in_list(txn_to_tpc) if count is not None: assert len(txn_to_tpc[0]) == count
def test_node_requests_missing_preprepare_malicious(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, malicious_setup, teardown): """ A node has bad network with primary and thus loses PRE-PREPARE, it requests PRE-PREPARE from non-primaries once it has sufficient PREPAREs but one of the non-primary does not send the PRE-PREPARE """ # primary_node = get_master_primary_node(txnPoolNodeSet) # slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node # other_nodes = [n for n in txnPoolNodeSet if n != slow_node] # bad_node = [n for n in other_nodes if n != primary_node][0] # good_non_primary_node = [n for n in other_nodes if n != slow_node # and n != bad_node and n != primary_node][0] primary_node, bad_node, good_non_primary_node, slow_node, other_nodes, \ bad_method, orig_method = malicious_setup slow_node.nodeIbStasher.delay(ppDelay(300, 0)) def get_reply_count_frm(node): return sum([ 1 for entry in slow_node.spylog.getAll(slow_node.process_message_rep) if entry.params['msg'].msg_type == PREPREPARE and entry.params['frm'] == node.name ]) old_reply_count_from_bad_node = get_reply_count_frm(bad_node) old_reply_count_from_good_node = get_reply_count_frm(good_non_primary_node) old_discarded = countDiscarded(slow_node.master_replica, 'does not have ' 'expected state') sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=10, num_batches=2) waitNodeDataEquality(looper, slow_node, *other_nodes) assert check_if_all_equal_in_list( [n.master_replica.ordered for n in txnPoolNodeSet]) assert not slow_node.master_replica.requested_pre_prepares if bad_method.__name__ == 'do_not_send': assert get_reply_count_frm(bad_node) == old_reply_count_from_bad_node else: assert countDiscarded(slow_node.master_replica, 'does not have expected state') > old_discarded assert get_reply_count_frm(good_non_primary_node) > \ old_reply_count_from_good_node slow_node.reset_delays_and_process_delayeds() bad_node.nodeMsgRouter.routes[MessageReq] = orig_method
def test_node_requests_missing_preprepare(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, teardown): """ A node has bad network with primary and thus loses PRE-PREPARE, it requests PRE-PREPARE from primary once it has sufficient PREPAREs """ slow_node, other_nodes, primary_node, other_non_primary_nodes = split_nodes( txnPoolNodeSet) # Delay PRE-PREPAREs by large amount simulating loss slow_node.nodeIbStasher.delay(ppDelay(300, 0)) old_count_pp = get_count( slow_node.master_replica, slow_node.master_replica._ordering_service.process_preprepare) old_count_mrq = { n.name: get_count(n, n.process_message_req) for n in other_nodes } old_count_mrp = get_count(slow_node, slow_node.process_message_rep) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=15, num_batches=5) waitNodeDataEquality(looper, slow_node, *other_nodes) assert not slow_node.master_replica.requested_pre_prepares # `slow_node` processed PRE-PREPARE # assert get_count(slow_node.master_replica, # slow_node.master_replica._ordering_service.process_preprepare) > old_count_pp # `slow_node` did receive `MessageRep` assert get_count(slow_node, slow_node.process_message_rep) > old_count_mrp # Primary node should received `MessageReq` and other nodes shouldn't recv_reqs = set() for n in other_non_primary_nodes: if get_count(n, n.process_message_req) > old_count_mrq[n.name]: recv_reqs.add(n.name) assert get_count(primary_node, primary_node.process_message_req) > \ old_count_mrq[primary_node.name] assert len(recv_reqs) == 0 # All nodes including the `slow_node` ordered the same requests assert check_if_all_equal_in_list( [n.master_replica._ordering_service.ordered for n in txnPoolNodeSet])
def test_node_requests_missing_preprepare(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, teardown): """ A node has bad network with primary and thus loses PRE-PREPARE, it requests PRE-PREPARE from primary once it has sufficient PREPAREs """ slow_node, other_nodes, primary_node, other_non_primary_nodes = split_nodes( txnPoolNodeSet) # Delay PRE-PREPAREs by large amount simulating loss slow_node.nodeIbStasher.delay(ppDelay(300, 0)) old_count_pp = get_count(slow_node.master_replica, slow_node.master_replica.processPrePrepare) old_count_mrq = {n.name: get_count(n, n.process_message_req) for n in other_nodes} old_count_mrp = get_count(slow_node, slow_node.process_message_rep) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=15, num_batches=5) waitNodeDataEquality(looper, slow_node, *other_nodes) assert not slow_node.master_replica.requested_pre_prepares # `slow_node` processed PRE-PREPARE assert get_count(slow_node.master_replica, slow_node.master_replica.processPrePrepare) > old_count_pp # `slow_node` did receive `MessageRep` assert get_count(slow_node, slow_node.process_message_rep) > old_count_mrp # Primary node should received `MessageReq` and other nodes shouldn't recv_reqs = set() for n in other_non_primary_nodes: if get_count(n, n.process_message_req) > old_count_mrq[n.name]: recv_reqs.add(n.name) assert get_count(primary_node, primary_node.process_message_req) > \ old_count_mrq[primary_node.name] assert len(recv_reqs) == 0 # All nodes including the `slow_node` ordered the same requests assert check_if_all_equal_in_list([n.master_replica.ordered for n in txnPoolNodeSet])
def tear(): # Repair any broken network for node in txnPoolNodeSet: node.reset_delays_and_process_delayeds() # Give a little time to process any delayed messages looper.runFor(3) # Check each node has same data ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) # Check each node has ordered all requests (no catchup) assert check_if_all_equal_in_list([n.master_replica.ordered for n in txnPoolNodeSet]) # Check the network is functional since all nodes reply sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5)
def tear(): # Repair any broken network for node in txnPoolNodeSet: node.reset_delays_and_process_delayeds() # Give a little time to process any delayed messages looper.runFor(3) # Check each node has same data ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) # Check each node has ordered all requests (no catchup) assert check_if_all_equal_in_list([n.master_replica.ordered for n in txnPoolNodeSet]) # Check the network is functional since all nodes reply send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 5)
def test_list_item_equality(): l = [ {'a': 1, 'b': 2, 'c': 3}, {'c': 3, 'a': 1, 'b': 2}, {'c': 3, 'a': 1, 'b': 2}, {'a': 1, 'b': 2, 'c': 3}, {'c': 3, 'a': 1, 'b': 2}, {'b': 2, 'c': 3, 'a': 1}, ] l1 = [{'a', 'b', 'c', 1}, {'c', 'a', 'b', 1}, {1, 'a', 'c', 'b'}] assert check_if_all_equal_in_list(l) assert check_if_all_equal_in_list(l1) assert check_if_all_equal_in_list([1, 1, 1, 1]) assert check_if_all_equal_in_list(['a', 'a', 'a', 'a']) assert not check_if_all_equal_in_list(['b', 'a', 'a', 'a']) assert not check_if_all_equal_in_list(l + [{'a': 1, 'b': 2, 'c': 33}]) assert not check_if_all_equal_in_list(l1 + [{'c', 'a', 'b', 11}])
def test_list_item_equality(): l = [ {'a': 1, 'b': 2, 'c': 3}, {'c': 3, 'a': 1, 'b': 2}, {'c': 3, 'a': 1, 'b': 2}, {'a': 1, 'b': 2, 'c': 3}, {'c': 3, 'a': 1, 'b': 2}, {'b': 2, 'c': 3, 'a': 1}, ] l1 = [{'a', 'b', 'c', 1}, {'c', 'a', 'b', 1}, {1, 'a', 'c', 'b'}] assert check_if_all_equal_in_list(l) assert check_if_all_equal_in_list(l1) assert check_if_all_equal_in_list([1, 1, 1, 1]) assert check_if_all_equal_in_list(['a', 'a', 'a', 'a']) assert not check_if_all_equal_in_list(['b', 'a', 'a', 'a']) assert not check_if_all_equal_in_list(l + [{'a': 1, 'b': 2, 'c': 33}]) assert not check_if_all_equal_in_list(l1 + [{'c', 'a', 'b', 11}])
def test_node_requests_missing_preprepare(looper, txnPoolNodeSet, client1, wallet1, client1Connected, teardown): """ A node has bad network with primary and thus loses PRE-PREPARE, it requests PRE-PREPARE from non-primaries once it has sufficient PREPAREs """ slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet) # Delay PRE-PREPAREs by large amount simulating loss slow_node.nodeIbStasher.delay(ppDelay(300, 0)) old_count_pp = get_count(slow_node.master_replica, slow_node.master_replica.processPrePrepare) old_count_mrq = { n.name: get_count(n, n.process_message_req) for n in other_nodes } old_count_mrp = get_count(slow_node, slow_node.process_message_rep) send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 15, 5) waitNodeDataEquality(looper, slow_node, *other_nodes) assert not slow_node.master_replica.requested_pre_prepares # `slow_node` processed PRE-PREPARE assert get_count(slow_node.master_replica, slow_node.master_replica.processPrePrepare) > old_count_pp # `slow_node` did receive `MessageRep` assert get_count(slow_node, slow_node.process_message_rep) > old_count_mrp # More than `f` nodes received `MessageReq` recv_reqs = set() for n in other_nodes: if get_count(n, n.process_message_req) > old_count_mrq[n.name]: recv_reqs.add(n.name) assert len(recv_reqs) > slow_node.f # All nodes including the `slow_node` ordered the same requests assert check_if_all_equal_in_list( [n.master_replica.ordered for n in txnPoolNodeSet])
def test_commits_recvd_first(looper, txnPoolNodeSet, client1, wallet1, client1Connected): slow_node = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)][-1] other_nodes = [n for n in txnPoolNodeSet if n != slow_node] delay = 50 slow_node.nodeIbStasher.delay(ppDelay(delay, 0)) slow_node.nodeIbStasher.delay(pDelay(delay, 0)) send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 20, 4) assert not slow_node.master_replica.prePrepares assert not slow_node.master_replica.prepares assert not slow_node.master_replica.commits assert len(slow_node.master_replica.commitsWaitingForPrepare) > 0 slow_node.reset_delays_and_process_delayeds() waitNodeDataEquality(looper, slow_node, *other_nodes) assert check_if_all_equal_in_list( [n.master_replica.ordered for n in txnPoolNodeSet]) assert slow_node.master_replica.prePrepares assert slow_node.master_replica.prepares assert slow_node.master_replica.commits assert not slow_node.master_replica.commitsWaitingForPrepare
def check_uncommitteds_equal(nodes): t_roots = [node.domainLedger.uncommittedRootHash for node in nodes] s_roots = [node.states[DOMAIN_LEDGER_ID].headHash for node in nodes] assert check_if_all_equal_in_list(t_roots) assert check_if_all_equal_in_list(s_roots) return t_roots[0], s_roots[0]
def check_last_3pc_master(node, other_nodes): last_3pc = [node.replicas[0].last_ordered_3pc] for n in other_nodes: last_3pc.append(n.replicas[0].last_ordered_3pc) assert check_if_all_equal_in_list(last_3pc)
def test_reverted_unordered(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ Before starting catchup, revert any uncommitted changes to state and ledger. This is to avoid any re-application of requests that were ordered but stashed Example scenario prepared (1, 4) start_view_change start_catchup ... .... ... committed and send Ordered (1, 2) ... .... preLedgerCatchUp force_process_ordered, take out (1,2) and stash (1, 2) now process stashed Ordered(1,2), its requests will be applied again Simulation: Delay COMMITs to a node so that it can not order requests but has prepared them. Then trigger a view change and make sure the slow node has not ordered same number of requests as others but has prepared so it can order when it receives COMMITs while view change is in progress. The slow node should revert unordered batches and but it should eventually process the ordered requests, so delay LEDGER_STATUS too so catchup is delayed """ slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node fast_nodes = [n for n in txnPoolNodeSet if n != slow_node] slow_node.nodeIbStasher.delay(cDelay(120, 0)) sent_batches = 5 sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2 * sent_batches, sent_batches) # Fast nodes have same last ordered and same data last_ordered = [n.master_last_ordered_3PC for n in fast_nodes] assert check_if_all_equal_in_list(last_ordered) ensure_all_nodes_have_same_data(looper, fast_nodes) # Slow nodes have different last ordered than fast nodes assert last_ordered[0] != slow_node.master_last_ordered_3PC # Delay LEDGER_STATUS so catchup starts late slow_node.nodeIbStasher.delay(lsDelay(100)) slow_node.nodeIbStasher.delay(msg_rep_delay(100)) slow_node.nodeIbStasher.delay(cr_delay(100)) # slow_node has not reverted batches assert sent_batches not in getAllReturnVals( slow_node.master_replica, slow_node.master_replica.revert_unordered_batches) ensure_view_change(looper, txnPoolNodeSet) def chk1(): # slow_node reverted all batches rv = getAllReturnVals( slow_node.master_replica, slow_node.master_replica.revert_unordered_batches) assert sent_batches in rv looper.run(eventually(chk1, retryWait=1)) # After the view change slow_node has prepared same requests as the fast # nodes have ordered assert last_ordered[ 0] == slow_node.master_replica.last_prepared_before_view_change # Deliver COMMITs slow_node.nodeIbStasher.reset_delays_and_process_delayeds(COMMIT) def chk2(): # slow_node stashed commits assert slow_node.master_replica.stasher.num_stashed_catchup == \ sent_batches * (len(txnPoolNodeSet) - 1) looper.run(eventually(chk2, retryWait=1)) # Deliver LEDGER_STATUS so catchup can complete slow_node.nodeIbStasher.reset_delays_and_process_delayeds(LEDGER_STATUS) slow_node.nodeIbStasher.reset_delays_and_process_delayeds(MESSAGE_RESPONSE) slow_node.nodeIbStasher.reset_delays_and_process_delayeds(CATCHUP_REP) # Ensure all nodes have same data ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) ensureElectionsDone(looper, txnPoolNodeSet) def chk3(): # slow_node processed stashed messages successfully assert slow_node.master_replica.stasher.num_stashed_catchup == 0 looper.run(eventually(chk3, retryWait=1)) # Ensure pool is functional sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 10, 2) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_reverted_unordered(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ Before starting catchup, revert any uncommitted changes to state and ledger. This is to avoid any re-application of requests that were ordered but stashed Example scenario prepared (1, 4) startViewChange start_catchup ... .... ... committed and send Ordered (1, 2) ... .... preLedgerCatchUp force_process_ordered, take out (1,2) and stash (1, 2) now process stashed Ordered(1,2), its requests will be applied again Simulation: Delay COMMITs to a node so that it can not order requests but has prepared them. Then trigger a view change and make sure the slow node has not ordered same number of requests as others but has prepared so it can order when it receives COMMITs while view change is in progress. The slow node should revert unordered batches and but it should eventually process the ordered requests, so delay LEDGER_STATUS too so catchup is delayed """ slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node fast_nodes = [n for n in txnPoolNodeSet if n != slow_node] slow_node.nodeIbStasher.delay(cDelay(120, 0)) sent_batches = 5 sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2 * sent_batches, sent_batches) # Fast nodes have same last ordered and same data last_ordered = [n.master_last_ordered_3PC for n in fast_nodes] assert check_if_all_equal_in_list(last_ordered) ensure_all_nodes_have_same_data(looper, fast_nodes) # Slow nodes have different last ordered than fast nodes assert last_ordered[0] != slow_node.master_last_ordered_3PC # Delay LEDGER_STATUS so catchup starts late slow_node.nodeIbStasher.delay(lsDelay(100)) slow_node.nodeIbStasher.delay(msg_rep_delay(100)) # slow_node has not reverted batches assert sent_batches not in getAllReturnVals( slow_node.master_replica, slow_node.master_replica.revert_unordered_batches) ensure_view_change(looper, txnPoolNodeSet) def chk1(): # slow_node reverted all batches rv = getAllReturnVals(slow_node.master_replica, slow_node.master_replica.revert_unordered_batches) assert sent_batches in rv looper.run(eventually(chk1, retryWait=1)) # After the view change slow_node has prepared same requests as the fast # nodes have ordered assert last_ordered[0] == slow_node.master_replica.last_prepared_before_view_change # Deliver COMMITs slow_node.nodeIbStasher.reset_delays_and_process_delayeds(COMMIT) def chk2(): # slow_node orders all requests as others have assert last_ordered[0] == slow_node.master_last_ordered_3PC looper.run(eventually(chk2, retryWait=1)) # Deliver LEDGER_STATUS so catchup can complete slow_node.nodeIbStasher.reset_delays_and_process_delayeds(LEDGER_STATUS) slow_node.nodeIbStasher.reset_delays_and_process_delayeds(MESSAGE_RESPONSE) # Ensure all nodes have same data ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) ensureElectionsDone(looper, txnPoolNodeSet) def chk3(): # slow_node processed stashed Ordered requests successfully rv = getAllReturnVals(slow_node, slow_node.processStashedOrderedReqs) assert sent_batches in rv looper.run(eventually(chk3, retryWait=1)) # Ensure pool is functional sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 10, 2) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def check_uncommitteds_equal(nodes): t_roots = [node.domainLedger.uncommittedRootHash for node in nodes] s_roots = [node.states[DOMAIN_LEDGER_ID].headHash for node in nodes] assert check_if_all_equal_in_list(t_roots) assert check_if_all_equal_in_list(s_roots) return t_roots[0], s_roots[0]
def check_last_3pc_master(node, other_nodes): last_3pc = [node.replicas[0].last_ordered_3pc] for n in other_nodes: last_3pc.append(n.replicas[0].last_ordered_3pc) assert check_if_all_equal_in_list(last_3pc)
def last_prepared(nodes): lst = [n.master_replica.last_prepared_certificate_in_view() for n in nodes] # All nodes have same last prepared assert check_if_all_equal_in_list(lst) return lst[0]