def test_node_requests_missing_preprepare_malicious(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, malicious_setup, teardown): """ A node has bad network with primary and thus loses PRE-PREPARE, it requests PRE-PREPARE from non-primaries once it has sufficient PREPAREs but one of the non-primary does not send the PRE-PREPARE """ # primary_node = get_master_primary_node(txnPoolNodeSet) # slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node # other_nodes = [n for n in txnPoolNodeSet if n != slow_node] # bad_node = [n for n in other_nodes if n != primary_node][0] # good_non_primary_node = [n for n in other_nodes if n != slow_node # and n != bad_node and n != primary_node][0] primary_node, bad_node, good_non_primary_node, slow_node, other_nodes, \ bad_method, orig_method = malicious_setup slow_node.nodeIbStasher.delay(ppDelay(300, 0)) def get_reply_count_frm(node): return sum([ 1 for entry in slow_node.spylog.getAll(slow_node.process_message_rep) if entry.params['msg'].msg_type == PREPREPARE and entry.params['frm'] == node.name ]) old_reply_count_from_bad_node = get_reply_count_frm(bad_node) old_reply_count_from_good_node = get_reply_count_frm(good_non_primary_node) old_discarded = countDiscarded(slow_node.master_replica, 'does not have ' 'expected state') sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=10, num_batches=2) waitNodeDataEquality(looper, slow_node, *other_nodes) assert check_if_all_equal_in_list( [n.master_replica.ordered for n in txnPoolNodeSet]) assert not slow_node.master_replica.requested_pre_prepares if bad_method.__name__ == 'do_not_send': assert get_reply_count_frm(bad_node) == old_reply_count_from_bad_node else: assert countDiscarded(slow_node.master_replica, 'does not have expected state') > old_discarded assert get_reply_count_frm(good_non_primary_node) > \ old_reply_count_from_good_node slow_node.reset_delays_and_process_delayeds() bad_node.nodeMsgRouter.routes[MessageReq] = orig_method
def fill_counters(nodes, log_message): global service_discard_counts service_discard_counts[log_message] = { n.master_replica._message_req_service.name: countDiscarded(n.master_replica._message_req_service, log_message) for n in nodes } node_discard_counts[log_message] = { n.name: countDiscarded(n, log_message) for n in nodes }
def test_catchup_with_ledger_statuses_in_old_format_from_one_node( txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath): """ A node is restarted and during a catch-up receives ledger statuses in an old format (without `protocolVersion`) from one of nodes in the pool. The test verifies that the node successfully completes the catch-up and participates in ordering of further transactions. """ node_to_restart = txnPoolNodeSet[-1] other_nodes = txnPoolNodeSet[:-1] old_node = txnPoolNodeSet[0] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) original_get_ledger_status = old_node.getLedgerStatus # Patch the method getLedgerStatus to # get_ledger_status_without_protocol_version for sending ledger status # in old format (without `protocolVersion`) def get_ledger_status_without_protocol_version(ledgerId: int): original_ledger_status = original_get_ledger_status(ledgerId) return LedgerStatusInOldFormat(original_ledger_status.ledgerId, original_ledger_status.txnSeqNo, original_ledger_status.viewNo, original_ledger_status.ppSeqNo, original_ledger_status.merkleRoot) old_node.getLedgerStatus = get_ledger_status_without_protocol_version # restart node disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_to_restart) looper.removeProdable(name=node_to_restart.name) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2) # add `node_to_restart` to pool node_to_restart = start_stopped_node(node_to_restart, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[-1] = node_to_restart looper.run(checkNodesConnected(txnPoolNodeSet)) # Verify that `node_to_restart` successfully completes catch-up waitNodeDataEquality(looper, node_to_restart, *other_nodes) # check discarding ledger statuses from `old_node` for all ledgers assert countDiscarded(node_to_restart, 'replied message has invalid structure') >= 3 # Verify that `node_to_restart` participates in ordering # of further transactions sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) waitNodeDataEquality(looper, node_to_restart, *other_nodes)
def chk(services, log_message, old_discard_counts=None): if old_discard_counts is None: global node_discard_counts old_discard_counts = node_discard_counts for s in services: assert countDiscarded( s, log_message) > old_discard_counts[log_message][s.name]
def test_master_primary_different_from_previous_view_for_itself( txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ After a view change, primary must be different from previous primary for master instance, it does not matter for other instance. Break it into 2 tests, one where the primary is malign and votes for itself but is still not made primary in the next view. """ old_view_no = checkViewNoForNodes(txnPoolNodeSet) pr = slow_primary(txnPoolNodeSet, 0, delay=10) old_pr_node = pr.node def _get_undecided_inst_id(self): undecideds = [i for i, r in self.replicas if r.isPrimary is None] # Try to nominate for the master instance return undecideds, 0 # Patching old primary's elector's method to nominate itself # again for the the new view old_pr_node.elector._get_undecided_inst_id = types.MethodType( _get_undecided_inst_id, old_pr_node.elector) # View change happens provoke_and_wait_for_view_change(looper, txnPoolNodeSet, old_view_no + 1, sdk_pool_handle, sdk_wallet_client) # Elections done ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) # New primary is not same as old primary assert getPrimaryReplica(txnPoolNodeSet, 0).node.name != old_pr_node.name # All other nodes discarded the nomination by the old primary for node in txnPoolNodeSet: if node != old_pr_node: assert countDiscarded(node.elector, 'of master in previous view too') == 1 # The new primary can still process requests sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5)
def test_master_primary_different_from_previous_view_for_itself( txnPoolNodeSet, looper, client1, wallet1): """ After a view change, primary must be different from previous primary for master instance, it does not matter for other instance. Break it into 2 tests, one where the primary is malign and votes for itself but is still not made primary in the next view. """ old_view_no = checkViewNoForNodes(txnPoolNodeSet) pr = slow_primary(txnPoolNodeSet, 0, delay=10) old_pr_node = pr.node def _get_undecided_inst_id(self): undecideds = [i for i, r in enumerate(self.replicas) if r.isPrimary is None] # Try to nominate for the master instance return undecideds, 0 # Patching old primary's elector's method to nominate itself # again for the the new view old_pr_node.elector._get_undecided_inst_id = types.MethodType( _get_undecided_inst_id, old_pr_node.elector) # View change happens provoke_and_wait_for_view_change(looper, txnPoolNodeSet, old_view_no + 1, wallet1, client1) # Elections done ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) # New primary is not same as old primary assert getPrimaryReplica(txnPoolNodeSet, 0).node.name != old_pr_node.name # All other nodes discarded the nomination by the old primary for node in txnPoolNodeSet: if node != old_pr_node: assert countDiscarded(node.elector, 'of master in previous view too') == 1 # The new primary can still process requests sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5)
def chk(nodes, log_message): global discard_counts for n in nodes: assert countDiscarded( n, log_message) > discard_counts[log_message][n.name]
def fill_counters(nodes, log_message): global discard_counts discard_counts[log_message] = { n.name: countDiscarded(n, log_message) for n in nodes }
def discardCounts(replicas, pat): counts = {} for r in replicas: counts[r.name] = countDiscarded(r, pat) return counts
def fill_counters(nodes, log_message): global discard_counts discard_counts[log_message] = {n.name: countDiscarded(n, log_message) for n in nodes}
def count_discarded(inst_id, count): for node in other_nodes: assert countDiscarded(node.replicas[inst_id], 'already ordered 3 phase message') == count
def discardCounts(checkpoint_services, pat): counts = {} for r in checkpoint_services: counts[str(r)] = countDiscarded(r, pat) return counts