def testViewChangesIfMasterPrimaryDisconnected(txnPoolNodeSet, looper, wallet1, client1, client1Connected, tconf): """ View change occurs when master's primary is disconnected """ # Setup nodes = txnPoolNodeSet viewNoBefore = checkViewNoForNodes(nodes) old_pr_node = get_master_primary_node(nodes) # Stop primary stopNodes([old_pr_node], looper) looper.removeProdable(old_pr_node) remainingNodes = list(set(nodes) - {old_pr_node}) # Sometimes it takes time for nodes to detect disconnection ensure_node_disconnected(looper, old_pr_node, remainingNodes, timeout=20) looper.runFor(tconf.ToleratePrimaryDisconnection + 2) # Give some time to detect disconnection and then verify that view has # changed and new primary has been elected waitForViewChange(looper, remainingNodes, viewNoBefore + 1) ensure_all_nodes_have_same_data(looper, nodes=remainingNodes) new_pr_node = get_master_primary_node(remainingNodes) assert old_pr_node != new_pr_node sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5)
def test_order_after_demote_and_restart(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tdir, tconf, allPluginsPath, sdk_wallet_stewards): sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 3, 3) primary_node = txnPoolNodeSet[0] node_to_stop = txnPoolNodeSet[1] node_to_demote = txnPoolNodeSet[2] txnPoolNodeSet.remove(node_to_demote) node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, txnPoolNodeSet, timeout=2) demote_node(looper, sdk_wallet_stewards[2], sdk_pool_handle, node_to_demote) config_helper = PNodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) txnPoolNodeSet[1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, check_primaries=False) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1, 1) def get_current_bls_keys(node): return node.master_replica._bls_bft_replica._bls_bft.bls_key_register._current_bls_keys assert get_current_bls_keys(restarted_node) == get_current_bls_keys(primary_node)
def testNodeCatchupAfterLostConnection(newNodeCaughtUp, txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns): """ A node that has poor internet connection and got unsynced after some transactions should eventually get the transactions which happened while it was not accessible :return: """ looper, newNode, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns logger.debug("Disconnecting node {}, ledger size {}". format(newNode, newNode.domainLedger.size)) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, newNode, stopNode=False) looper.removeProdable(newNode) # TODO: Check if the node has really stopped processing requests? logger.debug("Sending requests") sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) # Make sure new node got out of sync waitNodeDataInequality(looper, newNode, *txnPoolNodeSet[:-1]) # logger.debug("Ensure node {} gets disconnected".format(newNode)) ensure_node_disconnected(looper, newNode, txnPoolNodeSet[:-1]) logger.debug("Connecting the node {} back, ledger size {}". format(newNode, newNode.domainLedger.size)) looper.add(newNode) logger.debug("Waiting for the node to catch up, {}".format(newNode)) waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1]) logger.debug("Sending more requests") sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 10) checkNodeDataForEquality(newNode, *txnPoolNodeSet[:-1])
def test_order_after_demote_and_restart(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tdir, tconf, allPluginsPath, sdk_wallet_stewards): sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 3, 3) primary_node = txnPoolNodeSet[0] node_to_stop = txnPoolNodeSet[1] node_to_demote = txnPoolNodeSet[2] txnPoolNodeSet.remove(node_to_demote) node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, txnPoolNodeSet, timeout=2) demote_node(looper, sdk_wallet_stewards[2], sdk_pool_handle, node_to_demote) config_helper = PNodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) txnPoolNodeSet[1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1, 1) def get_current_bls_keys(node): return node.master_replica._bls_bft_replica._bls_bft.bls_key_register._current_bls_keys assert get_current_bls_keys(restarted_node) == get_current_bls_keys(primary_node)
def restart_nodes(looper, nodeSet, restart_set, tconf, tdir, allPluginsPath, after_restart_timeout=None, start_one_by_one=True, wait_for_elections=True): for node_to_stop in restart_set: node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) rest_nodes = [n for n in nodeSet if n not in restart_set] for node_to_stop in restart_set: ensure_node_disconnected(looper, node_to_stop, nodeSet, timeout=2) if after_restart_timeout: looper.runFor(after_restart_timeout) for node_to_restart in restart_set.copy(): config_helper = PNodeConfigHelper(node_to_restart.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_restart.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_restart.nodestack.ha, cliha=node_to_restart.clientstack.ha) looper.add(restarted_node) idx = nodeSet.index(node_to_restart) nodeSet[idx] = restarted_node idx = restart_set.index(node_to_restart) restart_set[idx] = restarted_node rest_nodes += [restarted_node] if start_one_by_one: looper.run(checkNodesConnected(rest_nodes)) if not start_one_by_one: looper.run(checkNodesConnected(nodeSet)) if wait_for_elections: ensureElectionsDone(looper=looper, nodes=nodeSet)
def test_state_regenerated_from_ledger(looper, nodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath): """ Node loses its state database but recreates it from ledger after start. Checking ATTRIB txns too since they store some data off ledger too """ trust_anchors = [] for i in range(5): trust_anchors.append(sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'TA' + str(i), TRUST_ANCHOR_STRING)) sdk_add_raw_attribute(looper, sdk_pool_handle, trust_anchors[-1], randomString(6), randomString(10)) for wh in trust_anchors: for i in range(3): sdk_add_new_nym(looper, sdk_pool_handle, wh, 'NP1' + str(i)) ensure_all_nodes_have_same_data(looper, nodeSet) node_to_stop = nodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv.db_path node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, nodeSet[:-1]) shutil.rmtree(state_db_path) config_helper = NodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode( node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) nodeSet[-1] = restarted_node looper.run(checkNodesConnected(nodeSet)) # Need some time as `last_ordered_3PC` is compared too and that is # communicated through catchup waitNodeDataEquality(looper, restarted_node, *nodeSet[:-1]) # Pool is still functional for wh in trust_anchors: sdk_add_new_nym(looper, sdk_pool_handle, wh, 'NP--' + randomString(5)) ensure_all_nodes_have_same_data(looper, nodeSet)
def restart_nodes(looper, nodeSet, restart_set, tconf, tdir, allPluginsPath, after_restart_timeout=None, per_add_timeout=None): for node_to_stop in restart_set: node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) rest_nodes = [n for n in nodeSet if n not in restart_set] for node_to_stop in restart_set: ensure_node_disconnected(looper, node_to_stop, nodeSet, timeout=2) if after_restart_timeout: looper.runFor(after_restart_timeout) for node_to_restart in restart_set: config_helper = PNodeConfigHelper(node_to_restart.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_restart.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_restart.nodestack.ha, cliha=node_to_restart.clientstack.ha) looper.add(restarted_node) idx = nodeSet.index(node_to_restart) nodeSet[idx] = restarted_node if per_add_timeout: looper.run(checkNodesConnected(rest_nodes + [restarted_node], customTimeout=per_add_timeout)) rest_nodes += [restarted_node] if not per_add_timeout: looper.run(checkNodesConnected(nodeSet, customTimeout=after_restart_timeout))
def test_state_regenerated_from_ledger(looper, tdirWithPoolTxns, tdirWithDomainTxnsUpdated, nodeSet, tconf, trustee, trusteeWallet, allPluginsPath): """ Node loses its state database but recreates it from ledger after start. Checking ATTRIB txns too since they store some data off ledger too """ trust_anchors = [] for i in range(5): trust_anchors.append(getClientAddedWithRole(nodeSet, tdirWithPoolTxns, looper, trustee, trusteeWallet, 'TA' + str(i), role=TRUST_ANCHOR)) addRawAttribute(looper, *trust_anchors[-1], randomString(6), randomString(10), dest=trust_anchors[-1][1].defaultId) for tc, tw in trust_anchors: for i in range(3): getClientAddedWithRole(nodeSet, tdirWithPoolTxns, looper, tc, tw, 'NP1' + str(i)) ensure_all_nodes_have_same_data(looper, nodeSet) node_to_stop = nodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv._dbPath node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop.name, nodeSet[:-1]) shutil.rmtree(state_db_path) restarted_node = TestNode(node_to_stop.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) nodeSet[-1] = restarted_node looper.run(checkNodesConnected(nodeSet)) # Need some time as `last_ordered_3PC` is compared too and that is # communicated through catchup waitNodeDataEquality(looper, restarted_node, *nodeSet[:-1]) # Pool is still functional for tc, tw in trust_anchors: getClientAddedWithRole(nodeSet, tdirWithPoolTxns, looper, tc, tw, 'NP--{}'.format(tc.name)) ensure_all_nodes_have_same_data(looper, nodeSet)
def test_state_regenerated_from_ledger(looper, nodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath): """ Node loses its state database but recreates it from ledger after start. Checking ATTRIB txns too since they store some data off ledger too """ endorsers = [] for i in range(5): endorsers.append(sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'TA' + str(i), ENDORSER_STRING)) sdk_add_raw_attribute(looper, sdk_pool_handle, endorsers[-1], randomString(6), randomString(10)) for wh in endorsers: for i in range(3): sdk_add_new_nym(looper, sdk_pool_handle, wh, 'NP1' + str(i)) ensure_all_nodes_have_same_data(looper, nodeSet) node_to_stop = nodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv.db_path node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, nodeSet[:-1]) shutil.rmtree(state_db_path) config_helper = NodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode( node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) nodeSet[-1] = restarted_node looper.run(checkNodesConnected(nodeSet)) # Need some time as `last_ordered_3PC` is compared too and that is # communicated through catchup waitNodeDataEquality(looper, restarted_node, *nodeSet[:-1]) # Pool is still functional for wh in endorsers: sdk_add_new_nym(looper, sdk_pool_handle, wh, 'NP--' + randomString(5)) ensure_all_nodes_have_same_data(looper, nodeSet)
def restart_nodes(looper, nodeSet, restart_set, tconf, tdir, allPluginsPath, after_restart_timeout=None, start_one_by_one=True, wait_for_elections=True): for node_to_stop in restart_set: node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) rest_nodes = [n for n in nodeSet if n not in restart_set] for node_to_stop in restart_set: ensure_node_disconnected(looper, node_to_stop, nodeSet, timeout=2) if after_restart_timeout: looper.runFor(after_restart_timeout) for node_to_restart in restart_set.copy(): config_helper = PNodeConfigHelper(node_to_restart.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_restart.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_restart.nodestack.ha, cliha=node_to_restart.clientstack.ha) looper.add(restarted_node) idx = nodeSet.index(node_to_restart) nodeSet[idx] = restarted_node restart_set[idx] = restarted_node rest_nodes += [restarted_node] if start_one_by_one: looper.run(checkNodesConnected(rest_nodes)) if not start_one_by_one: looper.run(checkNodesConnected(nodeSet)) if wait_for_elections: ensureElectionsDone(looper=looper, nodes=nodeSet)
def test_view_changes_if_master_primary_disconnected(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client, tdir, tconf, allPluginsPath): """ View change occurs when master's primary is disconnected """ # Setup nodes = txnPoolNodeSet old_view_no = checkViewNoForNodes(nodes) old_pr_node = get_master_primary_node(nodes) # Stop primary disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, old_pr_node, stopNode=True) looper.removeProdable(old_pr_node) remaining_nodes = list(set(nodes) - {old_pr_node}) # Sometimes it takes time for nodes to detect disconnection ensure_node_disconnected(looper, old_pr_node, remaining_nodes, timeout=20) looper.runFor(tconf.ToleratePrimaryDisconnection + 2) # Give some time to detect disconnection and then verify that view has # changed and new primary has been elected waitForViewChange(looper, remaining_nodes, old_view_no + 1) ensure_all_nodes_have_same_data(looper, nodes=remaining_nodes) new_pr_node = get_master_primary_node(remaining_nodes) assert old_pr_node != new_pr_node sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5) # Check if old primary can join the pool and still functions old_pr_node = start_stopped_node(old_pr_node, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet = remaining_nodes + [old_pr_node] looper.run( eventually(checkViewNoForNodes, txnPoolNodeSet, old_view_no + 1, timeout=tconf.VIEW_CHANGE_TIMEOUT)) # After node catches up it set view_no from audit ledger and do not need to do view_change assert len( getAllReturnVals(old_pr_node.view_changer, old_pr_node.view_changer.start_view_change, compare_val_to=True)) == 0 ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) assert not old_pr_node.view_changer._next_view_indications
def disconnect_node_and_ensure_disconnected(looper, poolNodes, disconnect: Union[str, TestNode], timeout=None, stopNode=True): if isinstance(disconnect, TestNode): disconnect = disconnect.name assert isinstance(disconnect, str) disconnectPoolNode(poolNodes, disconnect, stopNode=stopNode) ensure_node_disconnected(looper, disconnect, poolNodes, timeout=timeout)
def testViewChangesIfMasterPrimaryDisconnected(txnPoolNodeSet, looper, wallet1, client1, client1Connected, tconf, tdirWithPoolTxns, allPluginsPath): """ View change occurs when master's primary is disconnected """ # Setup nodes = txnPoolNodeSet old_view_no = checkViewNoForNodes(nodes) old_pr_node = get_master_primary_node(nodes) # Stop primary disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, old_pr_node, stopNode=True) looper.removeProdable(old_pr_node) remaining_nodes = list(set(nodes) - {old_pr_node}) # Sometimes it takes time for nodes to detect disconnection ensure_node_disconnected(looper, old_pr_node, remaining_nodes, timeout=20) looper.runFor(tconf.ToleratePrimaryDisconnection + 2) # Give some time to detect disconnection and then verify that view has # changed and new primary has been elected waitForViewChange(looper, remaining_nodes, old_view_no + 1) ensure_all_nodes_have_same_data(looper, nodes=remaining_nodes) new_pr_node = get_master_primary_node(remaining_nodes) assert old_pr_node != new_pr_node sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) # Check if old primary can join the pool and still functions old_pr_node = start_stopped_node(old_pr_node, looper, tconf, tdirWithPoolTxns, allPluginsPath) txnPoolNodeSet = remaining_nodes + [old_pr_node] looper.run( eventually(checkViewNoForNodes, txnPoolNodeSet, old_view_no + 1, timeout=10)) assert len( getAllReturnVals(old_pr_node, old_pr_node._start_view_change_if_possible, compare_val_to=True)) > 0 ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) assert not old_pr_node._next_view_indications
def test_restart_to_same_view_with_killed_primary(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): restart_timeout = tconf.ToleratePrimaryDisconnection + \ waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) primary = txnPoolNodeSet[0] alive_nodes = txnPoolNodeSet[1:] minority = alive_nodes[-1:] majority = alive_nodes[:-1] # Move to higher view by killing primary primary.cleanupOnStopping = True primary.stop() looper.removeProdable(primary) ensure_node_disconnected(looper, primary, txnPoolNodeSet) waitForViewChange(looper, alive_nodes, 1, customTimeout=VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper, alive_nodes, numInstances=3) # Add transaction to ledger sdk_send_random_and_check(looper, alive_nodes, sdk_pool_handle, sdk_wallet_client, 1) # Restart majority group majority_before_restart = majority.copy() restart_nodes(looper, alive_nodes, majority, tconf, tdir, allPluginsPath, after_restart_timeout=restart_timeout, start_one_by_one=False, wait_for_elections=False) waitForViewChange(looper, majority, 1, customTimeout=2.1 * VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper, majority, numInstances=3) # Check that nodes in minority group are aware that they might have inconsistent 3PC state for node in minority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 1 # Check that nodes in majority group didn't think they might have inconsistent 3PC state for node in majority_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes in majority group don't think they might have inconsistent 3PC state for node in majority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Restart minority group restart_nodes(looper, alive_nodes, minority, tconf, tdir, allPluginsPath, after_restart_timeout=restart_timeout, start_one_by_one=False, wait_for_elections=False) ensureElectionsDone(looper, alive_nodes, numInstances=3) # Check that all nodes are still functional sdk_ensure_pool_functional(looper, alive_nodes, sdk_wallet_client, sdk_pool_handle)
def test_restart_to_same_view_with_killed_primary(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): restart_timeout = tconf.ToleratePrimaryDisconnection + \ waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) primary = txnPoolNodeSet[0] alive_nodes = txnPoolNodeSet[1:] minority = alive_nodes[-1:] majority = alive_nodes[:-1] # Move to higher view by killing primary primary.cleanupOnStopping = True primary.stop() looper.removeProdable(primary) ensure_node_disconnected(looper, primary, txnPoolNodeSet) waitForViewChange(looper, alive_nodes, 1, customTimeout=VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper, alive_nodes, instances_list=range(3)) # Add transaction to ledger sdk_send_random_and_check(looper, alive_nodes, sdk_pool_handle, sdk_wallet_client, 1) # Restart majority group majority_before_restart = majority.copy() restart_nodes(looper, alive_nodes, majority, tconf, tdir, allPluginsPath, after_restart_timeout=restart_timeout, start_one_by_one=False, wait_for_elections=False) waitForViewChange(looper, majority, 1, customTimeout=2.1 * VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper, majority, instances_list=range(3)) # Check that nodes in minority group are aware that they might have inconsistent 3PC state for node in minority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 1 # Check that nodes in majority group didn't think they might have inconsistent 3PC state for node in majority_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes in majority group don't think they might have inconsistent 3PC state for node in majority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Restart minority group restart_nodes(looper, alive_nodes, minority, tconf, tdir, allPluginsPath, after_restart_timeout=restart_timeout, start_one_by_one=False, wait_for_elections=False) ensureElectionsDone(looper, alive_nodes, instances_list=range(3)) # Check that all nodes are still functional sdk_ensure_pool_functional(looper, alive_nodes, sdk_wallet_client, sdk_pool_handle)
def test_view_changes_if_master_primary_disconnected(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client, tdir, tconf, allPluginsPath): """ View change occurs when master's primary is disconnected """ # Setup nodes = txnPoolNodeSet old_view_no = checkViewNoForNodes(nodes) old_pr_node = get_master_primary_node(nodes) # Stop primary disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, old_pr_node, stopNode=True) looper.removeProdable(old_pr_node) remaining_nodes = list(set(nodes) - {old_pr_node}) # Sometimes it takes time for nodes to detect disconnection ensure_node_disconnected(looper, old_pr_node, remaining_nodes, timeout=20) looper.runFor(tconf.ToleratePrimaryDisconnection + 2) # Give some time to detect disconnection and then verify that view has # changed and new primary has been elected waitForViewChange(looper, remaining_nodes, old_view_no + 1) ensure_all_nodes_have_same_data(looper, nodes=remaining_nodes) new_pr_node = get_master_primary_node(remaining_nodes) assert old_pr_node != new_pr_node sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5) # Check if old primary can join the pool and still functions old_pr_node = start_stopped_node(old_pr_node, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet = remaining_nodes + [old_pr_node] looper.run(eventually(checkViewNoForNodes, txnPoolNodeSet, old_view_no + 1, timeout=tconf.VIEW_CHANGE_TIMEOUT)) assert len(getAllReturnVals(old_pr_node.view_changer, old_pr_node.view_changer._start_view_change_if_possible, compare_val_to=True)) > 0 ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) assert not old_pr_node.view_changer._next_view_indications
def disconnect_node_and_ensure_disconnected(looper: Looper, poolNodes: Iterable[TestNode], disconnect: Union[str, TestNode], timeout=None, stopNode=True): if isinstance(disconnect, TestNode): disconnect = disconnect.name assert isinstance(disconnect, str) matches = [n for n in poolNodes if n.name == disconnect] assert len(matches) == 1 node_to_disconnect = matches[0] disconnectPoolNode(poolNodes, disconnect, stopNode=stopNode) ensure_node_disconnected(looper, node_to_disconnect, set(poolNodes) - {node_to_disconnect}, timeout=timeout)
def disconnect_node_and_ensure_disconnected(looper: Looper, poolNodes: Iterable[TestNode], disconnect: Union[str, TestNode], timeout=None, stopNode=True): if isinstance(disconnect, TestNode): disconnect = disconnect.name assert isinstance(disconnect, str) matches = [n for n in poolNodes if n.name == disconnect] assert len(matches) == 1 node_to_disconnect = matches[0] disconnectPoolNode(poolNodes, disconnect, stopNode=stopNode) ensure_node_disconnected(looper, node_to_disconnect, set(poolNodes) - {node_to_disconnect}, timeout=timeout)
def test_state_recover_from_ledger(looper, tconf, tdir, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath, fees_set, mint_tokens, addresses, fees, do_post_node_creation, nodeSetWithIntegratedTokenPlugin, helpers): node_set = nodeSetWithIntegratedTokenPlugin current_amount = get_amount_from_token_txn(mint_tokens) seq_no = 1 current_amount, seq_no, _ = send_and_check_nym_with_fees( helpers, fees_set, seq_no, looper, addresses, current_amount) current_amount, seq_no, _ = send_and_check_transfer( helpers, [addresses[0], addresses[1]], fees, looper, current_amount, seq_no, transfer_summ=current_amount) current_amount, seq_no, _ = send_and_check_transfer( helpers, [addresses[1], addresses[2]], fees, looper, current_amount, seq_no, transfer_summ=current_amount) ensure_all_nodes_have_same_data(looper, node_set) node_to_stop = node_set[-1] state_db_pathes = [ state._kv.db_path for state in node_to_stop.states.values() ] node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, node_set[:-1]) for path in state_db_pathes: shutil.rmtree(path) config_helper = NodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) do_post_node_creation(restarted_node) looper.add(restarted_node) node_set = node_set[:-1] looper.run(checkNodesConnected(node_set)) waitNodeDataEquality(looper, restarted_node, *node_set[:-1]) ensure_all_nodes_have_same_data(looper, node_set) current_amount, seq_no, _ = send_and_check_transfer( helpers, [addresses[2], addresses[0]], fees, looper, current_amount, seq_no, transfer_summ=current_amount) current_amount, seq_no, _ = send_and_check_nym_with_fees( helpers, fees_set, seq_no, looper, addresses, current_amount) current_amount, seq_no, _ = send_and_check_nym_with_fees( helpers, fees_set, seq_no, looper, addresses, current_amount) ensure_all_nodes_have_same_data(looper, node_set)
def test_auth_txn_with_deprecated_key(tconf, tdir, allPluginsPath, txnPoolNodeSet, looper, sdk_wallet_trustee, sdk_pool_handle): """ Add to the auth_map a fake rule Send AUTH_RULE txn to change this fake rule (and set the fake key to the config state) Send GET_AUTH_RULE txn and check that the fake rule was changed Remove the fake auth rule from the map Check that we can't get the fake auth rule Restart the last node with its state regeneration Check that nodes data is equal after changing the existing auth rule (restarted node regenerate config state) """ fake_txn_type = "100002" fake_key = AuthActionAdd(txn_type=fake_txn_type, field="*", value="*").get_action_id() fake_constraint = one_trustee_constraint new_auth_constraint = AuthConstraint(role=STEWARD, sig_count=1, need_to_be_owner=False).as_dict # Add to the auth_map a fake rule with extend_auth_map(txnPoolNodeSet, fake_key, fake_constraint): # Send AUTH_RULE txn to change this fake rule (and set the fake key to the config state) sdk_send_and_check_auth_rule_request(looper, sdk_pool_handle, sdk_wallet_trustee, auth_action=ADD_PREFIX, auth_type=fake_txn_type, field='*', new_value='*', constraint=new_auth_constraint) # Send GET_AUTH_RULE txn and check that the fake rule was changed result = sdk_send_and_check_get_auth_rule_request( looper, sdk_pool_handle, sdk_wallet_trustee, auth_type=fake_txn_type, auth_action=ADD_PREFIX, field="*", new_value="*")[0][1]["result"][DATA][0] assert result[AUTH_TYPE] == fake_txn_type assert result[CONSTRAINT] == new_auth_constraint # Remove the fake auth rule from the map # Check that we can't get the fake auth rule with pytest.raises(RequestNackedException, match="not found in authorization map"): sdk_send_and_check_auth_rule_request( looper, sdk_pool_handle, sdk_wallet_trustee, auth_action=ADD_PREFIX, auth_type=fake_txn_type, field='*', new_value='*', constraint=AuthConstraint(role=STEWARD, sig_count=2, need_to_be_owner=False).as_dict) resp = sdk_send_and_check_get_auth_rule_request(looper, sdk_pool_handle, sdk_wallet_trustee) assert all(rule[AUTH_TYPE] != fake_txn_type for rule in resp[0][1]["result"][DATA]) with pytest.raises(RequestNackedException, match="not found in authorization map"): sdk_send_and_check_get_auth_rule_request(looper, sdk_pool_handle, sdk_wallet_trustee, auth_type=fake_txn_type, auth_action=ADD_PREFIX, field="*", new_value="*") # Restart the last node with its state regeneration ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) node_to_stop = txnPoolNodeSet[-1] node_state = node_to_stop.states[CONFIG_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv.db_path node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, txnPoolNodeSet[:-1]) shutil.rmtree(state_db_path) config_helper = NodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) txnPoolNodeSet[-1] = restarted_node # Check that nodes data is equal (restarted node regenerate config state) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30) sdk_send_and_check_auth_rule_request(looper, sdk_pool_handle, sdk_wallet_trustee, auth_action=ADD_PREFIX, auth_type=NYM, field=ROLE, new_value=STEWARD, constraint=AuthConstraint( role=STEWARD, sig_count=2, need_to_be_owner=False).as_dict) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=20)
def test_new_node_catchup_update_projection(looper, nodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath, some_transactions_done): """ A node which receives txns from catchup updates both ledger and projection 4 nodes start up and some txns happen, after txns are done, new node joins and starts catching up, the node should not process requests while catchup is in progress. Make sure the new requests are coming from the new NYMs added while the node was offline or catching up. """ # Create a new node and stop it. new_steward_wallet, new_node = sdk_node_theta_added(looper, nodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath, node_config_helper_class=NodeConfigHelper, testNodeClass=TestNode) waitNodeDataEquality(looper, new_node, *nodeSet[:-1]) ta_count = 2 np_count = 2 new_txn_count = 2 * ta_count + np_count # Since ATTRIB txn is done for TA old_ledger_sizes = {} new_ledger_sizes = {} old_projection_sizes = {} new_projection_sizes = {} old_seq_no_map_sizes = {} new_seq_no_map_sizes = {} def get_ledger_size(node): return len(node.domainLedger) def get_projection_size(node): domain_state = node.getState(DOMAIN_LEDGER_ID) return len(domain_state.as_dict) def get_seq_no_map_size(node): return node.seqNoDB.size def fill_counters(ls, ps, ss, nodes): for n in nodes: ls[n.name] = get_ledger_size(n) ps[n.name] = get_projection_size(n) ss[n.name] = get_seq_no_map_size(n) def check_sizes(nodes): for node in nodes: assert new_ledger_sizes[node.name] - \ old_ledger_sizes[node.name] == new_txn_count assert new_projection_sizes[node.name] - \ old_projection_sizes[node.name] == new_txn_count assert new_seq_no_map_sizes[node.name] - \ old_seq_no_map_sizes[node.name] == new_txn_count # Stop a node and note down the sizes of ledger and projection (state) other_nodes = nodeSet[:-1] fill_counters(old_ledger_sizes, old_projection_sizes, old_seq_no_map_sizes, other_nodes) new_node.cleanupOnStopping = False new_node.stop() looper.removeProdable(new_node) ensure_node_disconnected(looper, new_node, other_nodes) trust_anchors = [] attributes = [] for i in range(ta_count): trust_anchors.append( sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, role='TRUST_ANCHOR', alias='TA' + str(i))) attributes.append((randomString(6), randomString(10))) sdk_add_raw_attribute(looper, sdk_pool_handle, trust_anchors[-1], *attributes[-1]) non_privileged = [] for i in range(np_count): non_privileged.append( sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, alias='NP' + str(i))) checkNodeDataForEquality(nodeSet[0], *other_nodes) fill_counters(new_ledger_sizes, new_projection_sizes, new_seq_no_map_sizes, other_nodes) # The size difference should be same as number of new NYM txns check_sizes(other_nodes) config_helper = NodeConfigHelper(new_node.name, tconf, chroot=tdir) new_node = TestNode( new_node.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=new_node.nodestack.ha, cliha=new_node.clientstack.ha) looper.add(new_node) nodeSet[-1] = new_node fill_counters(old_ledger_sizes, old_projection_sizes, old_seq_no_map_sizes, [new_node]) looper.run(checkNodesConnected(nodeSet)) waitNodeDataEquality(looper, new_node, *other_nodes) fill_counters(new_ledger_sizes, new_projection_sizes, new_seq_no_map_sizes, [new_node]) check_sizes([new_node]) # Set the old counters to be current ledger and projection size fill_counters(old_ledger_sizes, old_projection_sizes, old_seq_no_map_sizes, nodeSet) more_nyms_count = 2 for wh in trust_anchors: for i in range(more_nyms_count): non_privileged.append(sdk_add_new_nym(looper, sdk_pool_handle, wh, alias='NP1' + str(i))) # The new node should process transactions done by Nyms added to its # ledger while catchup fill_counters(new_ledger_sizes, new_projection_sizes, new_seq_no_map_sizes, nodeSet) new_txn_count = more_nyms_count * len(trust_anchors) check_sizes(nodeSet)
def test_new_node_catchup_update_projection(looper, tdirWithPoolTxns, tdirWithDomainTxnsUpdated, nodeSet, tconf, trustee, trusteeWallet, allPluginsPath, some_transactions_done): """ A node which receives txns from catchup updates both ledger and projection 4 nodes start up and some txns happen, after txns are done, new node joins and starts catching up, the node should not process requests while catchup is in progress. Make sure the new requests are coming from the new NYMs added while the node was offline or catching up. """ # Create a new node and stop it. new_steward, new_steward_wallet, new_node = nodeThetaAdded( looper, nodeSet, tdirWithPoolTxns, tconf, trustee, trusteeWallet, allPluginsPath, TestNode, TestClient, tdirWithPoolTxns) waitNodeDataEquality(looper, new_node, *nodeSet[:-1]) ta_count = 2 np_count = 2 new_txn_count = 2 * ta_count + np_count # Since ATTRIB txn is done for TA old_ledger_sizes = {} new_ledger_sizes = {} old_projection_sizes = {} new_projection_sizes = {} old_seq_no_map_sizes = {} new_seq_no_map_sizes = {} def get_ledger_size(node): return len(node.domainLedger) def get_projection_size(node): domain_state = node.getState(DOMAIN_LEDGER_ID) return len(domain_state.as_dict) def get_seq_no_map_size(node): return node.seqNoDB.size def fill_counters(ls, ps, ss, nodes): for n in nodes: ls[n.name] = get_ledger_size(n) ps[n.name] = get_projection_size(n) ss[n.name] = get_seq_no_map_size(n) def check_sizes(nodes): for node in nodes: assert new_ledger_sizes[node.name] - \ old_ledger_sizes[node.name] == new_txn_count assert new_projection_sizes[node.name] - \ old_projection_sizes[node.name] == new_txn_count assert new_seq_no_map_sizes[node.name] - \ old_seq_no_map_sizes[node.name] == new_txn_count # Stop a node and note down the sizes of ledger and projection (state) other_nodes = nodeSet[:-1] fill_counters(old_ledger_sizes, old_projection_sizes, old_seq_no_map_sizes, other_nodes) new_node.cleanupOnStopping = False new_node.stop() looper.removeProdable(new_node) ensure_node_disconnected(looper, new_node.name, other_nodes) trust_anchors = [] attributes = [] for i in range(ta_count): trust_anchors.append( getClientAddedWithRole(other_nodes, tdirWithPoolTxns, looper, trustee, trusteeWallet, 'TA' + str(i), role=TRUST_ANCHOR, client_connects_to=len(other_nodes))) attributes.append((randomString(6), randomString(10))) addRawAttribute(looper, *trust_anchors[-1], *attributes[-1], dest=trust_anchors[-1][1].defaultId) non_privileged = [] for i in range(np_count): non_privileged.append( getClientAddedWithRole(other_nodes, tdirWithPoolTxns, looper, trustee, trusteeWallet, 'NP' + str(i), client_connects_to=len(other_nodes))) checkNodeDataForEquality(nodeSet[0], *other_nodes) fill_counters(new_ledger_sizes, new_projection_sizes, new_seq_no_map_sizes, other_nodes) # The size difference should be same as number of new NYM txns check_sizes(other_nodes) new_node = TestNode(new_node.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=new_node.nodestack.ha, cliha=new_node.clientstack.ha) looper.add(new_node) nodeSet[-1] = new_node fill_counters(old_ledger_sizes, old_projection_sizes, old_seq_no_map_sizes, [new_node]) looper.run(checkNodesConnected(nodeSet)) waitNodeDataEquality(looper, new_node, *other_nodes) fill_counters(new_ledger_sizes, new_projection_sizes, new_seq_no_map_sizes, [new_node]) check_sizes([new_node]) for i, (tc, tw) in enumerate(trust_anchors): reply = getAttribute(looper, tc, tw, tw.defaultId, *attributes[i]) all_replies = tc.getRepliesFromAllNodes(reply[f.IDENTIFIER.nm], reply[f.REQ_ID.nm]) assertLength(all_replies, len(nodeSet)) assert new_node.clientstack.name in all_replies # Set the old counters to be current ledger and projection size fill_counters(old_ledger_sizes, old_projection_sizes, old_seq_no_map_sizes, nodeSet) more_nyms_count = 2 for tc, tw in trust_anchors: for i in range(more_nyms_count): non_privileged.append( getClientAddedWithRole(other_nodes, tdirWithPoolTxns, looper, tc, tw, 'NP1' + str(i))) # The new node should process transactions done by Nyms added to its # ledger while catchup fill_counters(new_ledger_sizes, new_projection_sizes, new_seq_no_map_sizes, nodeSet) new_txn_count = more_nyms_count * len(trust_anchors) check_sizes(nodeSet)
def disconnect_node(looper, node, other_nodes): node.stop() looper.removeProdable(node) ensure_node_disconnected(looper, node, other_nodes) check_if_pool_n_minus_f(other_nodes)
def test_new_node_catchup_update_projection(looper, tdirWithPoolTxns, tdirWithDomainTxnsUpdated, nodeSet, tconf, trustee, trusteeWallet, allPluginsPath, some_transactions_done ): """ A node which receives txns from catchup updates both ledger and projection 4 nodes start up and some txns happen, after txns are done, new node joins and starts catching up, the node should not process requests while catchup is in progress. Make sure the new requests are coming from the new NYMs added while the node was offline or catching up. """ # Create a new node and stop it. new_steward, new_steward_wallet, new_node = nodeThetaAdded(looper, nodeSet, tdirWithPoolTxns, tconf, trustee, trusteeWallet, allPluginsPath, TestNode, TestClient, tdirWithPoolTxns) waitNodeDataEquality(looper, new_node, *nodeSet[:-1]) ta_count = 2 np_count = 2 new_txn_count = 2*ta_count + np_count # Since ATTRIB txn is done for TA old_ledger_sizes = {} new_ledger_sizes = {} old_projection_sizes = {} new_projection_sizes = {} old_seq_no_map_sizes = {} new_seq_no_map_sizes = {} def get_ledger_size(node): return len(node.domainLedger) def get_projection_size(node): domain_state = node.getState(DOMAIN_LEDGER_ID) return len(domain_state.as_dict) def get_seq_no_map_size(node): return node.seqNoDB.size def fill_counters(ls, ps, ss, nodes): for n in nodes: ls[n.name] = get_ledger_size(n) ps[n.name] = get_projection_size(n) ss[n.name] = get_seq_no_map_size(n) def check_sizes(nodes): for node in nodes: assert new_ledger_sizes[node.name] - old_ledger_sizes[node.name] == new_txn_count assert new_projection_sizes[node.name] - old_projection_sizes[node.name] == new_txn_count assert new_seq_no_map_sizes[node.name] - old_seq_no_map_sizes[node.name] == new_txn_count # Stop a node and note down the sizes of ledger and projection (state) other_nodes = nodeSet[:-1] fill_counters(old_ledger_sizes, old_projection_sizes, old_seq_no_map_sizes, other_nodes) new_node.cleanupOnStopping = False new_node.stop() looper.removeProdable(new_node) ensure_node_disconnected(looper, new_node.name, other_nodes) trust_anchors = [] attributes = [] for i in range(ta_count): trust_anchors.append(getClientAddedWithRole(other_nodes, tdirWithPoolTxns, looper, trustee, trusteeWallet, 'TA'+str(i), role=TRUST_ANCHOR, client_connects_to=len(other_nodes))) attributes.append((randomString(6), randomString(10))) addRawAttribute(looper, *trust_anchors[-1], *attributes[-1], dest=trust_anchors[-1][1].defaultId) non_privileged = [] for i in range(np_count): non_privileged.append(getClientAddedWithRole(other_nodes, tdirWithPoolTxns, looper, trustee, trusteeWallet, 'NP'+str(i), client_connects_to=len(other_nodes))) checkNodeDataForEquality(nodeSet[0], *other_nodes) fill_counters(new_ledger_sizes, new_projection_sizes, new_seq_no_map_sizes, other_nodes) # The size difference should be same as number of new NYM txns check_sizes(other_nodes) new_node = TestNode(new_node.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=new_node.nodestack.ha, cliha=new_node.clientstack.ha) looper.add(new_node) nodeSet[-1] = new_node fill_counters(old_ledger_sizes, old_projection_sizes, old_seq_no_map_sizes, [new_node]) looper.run(checkNodesConnected(nodeSet)) waitNodeDataEquality(looper, new_node, *other_nodes) fill_counters(new_ledger_sizes, new_projection_sizes, new_seq_no_map_sizes, [new_node]) check_sizes([new_node]) for i, (tc, tw) in enumerate(trust_anchors): reply = getAttribute(looper, tc, tw, tw.defaultId, *attributes[i]) all_replies = tc.getRepliesFromAllNodes(reply[f.IDENTIFIER.nm], reply[f.REQ_ID.nm]) assertLength(all_replies, len(nodeSet)) assert new_node.clientstack.name in all_replies # Set the old counters to be current ledger and projection size fill_counters(old_ledger_sizes, old_projection_sizes, old_seq_no_map_sizes, nodeSet) more_nyms_count = 2 for tc, tw in trust_anchors: for i in range(more_nyms_count): non_privileged.append(getClientAddedWithRole(other_nodes, tdirWithPoolTxns, looper, tc, tw, 'NP1' + str(i))) # The new node should process transactions done by Nyms added to its # ledger while catchup fill_counters(new_ledger_sizes, new_projection_sizes, new_seq_no_map_sizes, nodeSet) new_txn_count = more_nyms_count*len(trust_anchors) check_sizes(nodeSet)
def test_state_recovery_with_xfer(looper, tconf, tdir, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath, do_post_node_creation, nodeSetWithIntegratedTokenPlugin, helpers, valid_upgrade, mint_tokens, addresses, fees_set, fees, monkeypatch): version1 = "1.1.50" version2 = "1.1.88" current_amount = get_amount_from_token_txn(mint_tokens) seq_no = 1 node_set = nodeSetWithIntegratedTokenPlugin current_amount, seq_no, _ = send_and_check_nym_with_fees(helpers, fees_set, seq_no, looper, addresses, current_amount) # send POOL_UPGRADE to write in a ledger last_ordered = node_set[0].master_last_ordered_3PC[1] sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, valid_upgrade) looper.run(eventually(lambda: assertEquality(node_set[0].master_last_ordered_3PC[1], last_ordered + 1))) send_node_upgrades(node_set, version1, looper) for n in node_set: handler = n.write_manager.request_handlers.get(XFER_PUBLIC)[0] handler_for_1_0_0 = n.write_manager._request_handlers_with_version.get((XFER_PUBLIC, "1.0.0"))[0] monkeypatch.setattr(handler, 'update_state', handler_for_1_0_0.update_state) current_amount, seq_no, _ = send_and_check_transfer(helpers, [addresses[0], addresses[1]], fees_set, looper, current_amount, seq_no, transfer_summ=current_amount) send_node_upgrades(node_set, version2, looper) monkeypatch.undo() current_amount, seq_no, _ = send_and_check_transfer(helpers, [addresses[1], addresses[0]], fees_set, looper, current_amount, seq_no, transfer_summ=current_amount) node_to_stop = node_set[-1] state_db_pathes = [state._kv.db_path for state in node_to_stop.states.values()] node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, node_set[:-1]) for path in state_db_pathes: shutil.rmtree(path) config_helper = NodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode( node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) do_post_node_creation(restarted_node) looper.add(restarted_node) node_set[-1] = restarted_node looper.run(checkNodesConnected(node_set)) waitNodeDataEquality(looper, restarted_node, *node_set[:-1], exclude_from_check=['check_last_ordered_3pc_backup']) current_amount, seq_no, _ = send_and_check_transfer(helpers, [addresses[0], addresses[1]], {}, looper, current_amount, seq_no, transfer_summ=1) waitNodeDataEquality(looper, restarted_node, *node_set[:-1], exclude_from_check=['check_last_ordered_3pc_backup'])
def disconnect_node(looper, node, other_nodes): node.stop() looper.removeProdable(node) ensure_node_disconnected(looper, node, other_nodes) check_if_pool_n_minus_f(other_nodes)
def test_state_regenerated_from_ledger(looper, tdirWithClientPoolTxns, tdirWithDomainTxnsUpdated, nodeSet, tconf, tdir, trustee, trusteeWallet, allPluginsPath): """ Node loses its state database but recreates it from ledger after start. Checking ATTRIB txns too since they store some data off ledger too """ trust_anchors = [] for i in range(5): trust_anchors.append( getClientAddedWithRole(nodeSet, tdirWithClientPoolTxns, looper, trustee, trusteeWallet, 'TA' + str(i), role=TRUST_ANCHOR)) addRawAttribute(looper, *trust_anchors[-1], randomString(6), randomString(10), dest=trust_anchors[-1][1].defaultId) for tc, tw in trust_anchors: for i in range(3): getClientAddedWithRole(nodeSet, tdirWithClientPoolTxns, looper, tc, tw, 'NP1' + str(i)) ensure_all_nodes_have_same_data(looper, nodeSet) node_to_stop = nodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv.db_path node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop.name, nodeSet[:-1]) shutil.rmtree(state_db_path) config_helper = NodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) nodeSet[-1] = restarted_node looper.run(checkNodesConnected(nodeSet)) # Need some time as `last_ordered_3PC` is compared too and that is # communicated through catchup waitNodeDataEquality(looper, restarted_node, *nodeSet[:-1]) # Pool is still functional for tc, tw in trust_anchors: getClientAddedWithRole(nodeSet, tdirWithClientPoolTxns, looper, tc, tw, 'NP--{}'.format(tc.name)) ensure_all_nodes_have_same_data(looper, nodeSet)