def testNodeConnection(allPluginsPath, tconf, tdir, tdir_for_func, tconf_for_func, looper, txnPoolNodeSetNotStarted): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) nodes = txnPoolNodeSetNotStarted[:2] for node in nodes: tellKeysToOthers(node, nodes) A, B = nodes looper.add(A) looper.runFor(4) logger.debug("wait done") looper.add(B) looper.runFor(4) looper.run(checkNodesConnected([A, B])) looper.stopall() looper.removeProdable(A) looper.removeProdable(B) A = start_stopped_node(A, looper, tconf, tdir, allPluginsPath) looper.runFor(4) B = start_stopped_node(B, looper, tconf, tdir, allPluginsPath) looper.run(checkNodesConnected([A, B])) for node in txnPoolNodeSetNotStarted[2:]: looper.add(node) all_nodes = [A, B] + txnPoolNodeSetNotStarted[2:] looper.run(checkNodesConnected(all_nodes)) stopNodes(all_nodes, looper) for node in all_nodes: looper.removeProdable(node)
def restart_nodes(looper, nodeSet, restart_set, tconf, tdir, allPluginsPath, after_restart_timeout=None, start_one_by_one=True, wait_for_elections=True): for node_to_stop in restart_set: node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) rest_nodes = [n for n in nodeSet if n not in restart_set] for node_to_stop in restart_set: ensure_node_disconnected(looper, node_to_stop, nodeSet, timeout=2) if after_restart_timeout: looper.runFor(after_restart_timeout) for node_to_restart in restart_set.copy(): config_helper = PNodeConfigHelper(node_to_restart.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_restart.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_restart.nodestack.ha, cliha=node_to_restart.clientstack.ha) looper.add(restarted_node) idx = nodeSet.index(node_to_restart) nodeSet[idx] = restarted_node idx = restart_set.index(node_to_restart) restart_set[idx] = restarted_node rest_nodes += [restarted_node] if start_one_by_one: looper.run(checkNodesConnected(rest_nodes)) if not start_one_by_one: looper.run(checkNodesConnected(nodeSet)) if wait_for_elections: ensureElectionsDone(looper=looper, nodes=nodeSet)
def testChangeHaPersistsPostNodesRestart(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward): new_steward_wallet, new_node = \ sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, 'AnotherSteward' + randomString(4), 'AnotherNode' + randomString(4), tdir, tconf) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) node_new_ha, client_new_ha = genHa(2) logger.debug("{} changing HAs to {} {}".format(new_node, node_new_ha, client_new_ha)) # Making the change HA txn an confirming its succeeded node_dest = hexToFriendly(new_node.nodestack.verhex) sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, node_dest, new_node.name, node_new_ha.host, node_new_ha.port, client_new_ha.host, client_new_ha.port) # Stopping existing nodes for node in txnPoolNodeSet: node.stop() looper.removeProdable(node) # Starting nodes again by creating `Node` objects since that simulates # what happens when starting the node with script restartedNodes = [] for node in txnPoolNodeSet[:-1]: config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=node.nodestack.ha, cliha=node.clientstack.ha) looper.add(restartedNode) restartedNodes.append(restartedNode) # Starting the node whose HA was changed config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir) node = TestNode(new_node.name, config_helper=config_helper, config=tconf, ha=node_new_ha, cliha=client_new_ha) looper.add(node) restartedNodes.append(node) looper.run(checkNodesConnected(restartedNodes)) waitNodeDataEquality(looper, node, *restartedNodes[:-1]) sdk_pool_refresh(looper, sdk_pool_handle) sdk_ensure_pool_functional(looper, restartedNodes, sdk_wallet_client, sdk_pool_handle)
def testNodesComingUpAtDifferentTimes(allPluginsPath, tconf, tdir, tdir_for_func, tconf_for_func, looper, txnPoolNodeSetNotStarted): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) nodes = txnPoolNodeSetNotStarted names = list(node.name for node in nodes) shuffle(names) waits = [randint(1, 10) for _ in names] rwaits = [randint(1, 10) for _ in names] for node in nodes: tellKeysToOthers(node, nodes) for i, node in enumerate(nodes): looper.add(node) looper.runFor(waits[i]) looper.run(checkNodesConnected(nodes)) logger.debug("connects") logger.debug("node order: {}".format(names)) logger.debug("waits: {}".format(waits)) current_node_set = set(nodes) for node in nodes: disconnect_node_and_ensure_disconnected(looper, current_node_set, node, timeout=len(nodes), stopNode=True) looper.removeProdable(node) current_node_set.remove(node) for i, node in enumerate(nodes): restarted_node = start_stopped_node(node, looper, tconf, tdir, allPluginsPath) current_node_set.add(restarted_node) looper.runFor(rwaits[i]) looper.runFor(3) looper.run(checkNodesConnected(current_node_set)) stopNodes(current_node_set, looper) logger.debug("reconnects") logger.debug("node order: {}".format(names)) logger.debug("rwaits: {}".format(rwaits)) for node in current_node_set: looper.removeProdable(node)
def testKeyShareParty(looper, txnPoolNodeSet, tdir_for_func, tconf_for_func): """ connections to all nodes should be successfully established when key sharing is enabled. """ logger.debug("-----sharing keys-----") looper.run(checkNodesConnected(txnPoolNodeSet)) logger.debug("-----key sharing done, connect after key sharing-----") looper.run(checkNodesConnected(txnPoolNodeSet), msgAll(txnPoolNodeSet)) for node in txnPoolNodeSet: node.stop() looper.removeProdable(node)
def testNodeKeysChanged(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, nodeThetaAdded, allPluginsPath=None): newSteward, newStewardWallet, newNode = nodeThetaAdded # Since the node returned by fixture `nodeThetaAdded` was abandoned in the # previous test, so getting node `Theta` from `txnPoolNodeSet` newNode = getNodeWithName(txnPoolNodeSet, newNode.name) newNode.stop() nodeHa, nodeCHa = HA(*newNode.nodestack.ha), HA(*newNode.clientstack.ha) sigseed = randomString(32).encode() verkey = SimpleSigner(seed=sigseed).naclSigner.verhex.decode() changeNodeKeys(looper, newSteward, newStewardWallet, newNode, verkey) initLocalKeep(newNode.name, tdirWithPoolTxns, sigseed) initLocalKeep(newNode.name+CLIENT_STACK_SUFFIX, tdirWithPoolTxns, sigseed) looper.removeProdable(name=newNode.name) logger.debug("{} starting with HAs {} {}".format(newNode, nodeHa, nodeCHa)) node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(node) # The last element of `txnPoolNodeSet` is the node Theta that was just # stopped txnPoolNodeSet[-1] = node looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run(eventually(checkNodeLedgersForEquality, node, *txnPoolNodeSet[:-1], retryWait=1, timeout=10)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def testNodeRequestingTxns(txnPoolNodeSet, nodeCreatedAfterSomeTxns): """ A newly joined node is catching up and sends catchup requests to other nodes but one of the nodes does not reply and the newly joined node cannot complete the process till the timeout and then requests the missing transactions. """ looper, newNode, client, wallet, _, _ = nodeCreatedAfterSomeTxns # So nodes wont tell the clients about the newly joined node so they # dont send any request to the newly joined node for node in txnPoolNodeSet: node.sendPoolInfoToClients = types.MethodType(lambda x, y: None, node) txnPoolNodeSet.append(newNode) def ignoreCatchupReq(self, req, frm): logger.info("{} being malicious and ignoring catchup request {} " "from {}".format(self, req, frm)) # One of the node does not process catchup request. txnPoolNodeSet[0].nodeMsgRouter.routes[CatchupReq] = types.MethodType( ignoreCatchupReq, txnPoolNodeSet[0].ledgerManager) sendRandomRequests(wallet, client, 10) looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=60)) looper.run(eventually(checkNodeLedgersForEquality, newNode, *txnPoolNodeSet[:-1], retryWait=1, timeout=90))
def test_order_after_demote_and_restart(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tdir, tconf, allPluginsPath, sdk_wallet_stewards): sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 3, 3) primary_node = txnPoolNodeSet[0] node_to_stop = txnPoolNodeSet[1] node_to_demote = txnPoolNodeSet[2] txnPoolNodeSet.remove(node_to_demote) node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, txnPoolNodeSet, timeout=2) demote_node(looper, sdk_wallet_stewards[2], sdk_pool_handle, node_to_demote) config_helper = PNodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) txnPoolNodeSet[1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, check_primaries=False) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1, 1) def get_current_bls_keys(node): return node.master_replica._bls_bft_replica._bls_bft.bls_key_register._current_bls_keys assert get_current_bls_keys(restarted_node) == get_current_bls_keys(primary_node)
def update_node_data_and_reconnect(looper, txnPoolNodeSet, steward_wallet, sdk_pool_handle, node, new_node_ip, new_node_port, new_client_ip, new_client_port, tdir, tconf): node_ha = node.nodestack.ha cli_ha = node.clientstack.ha node_dest = hexToFriendly(node.nodestack.verhex) sdk_send_update_node(looper, steward_wallet, sdk_pool_handle, node_dest, node.name, new_node_ip, new_node_port, new_client_ip, new_client_port) # restart the Node with new HA node.stop() looper.removeProdable(name=node.name) config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) restartedNode = TestNode(node.name, config_helper=config_helper, config=tconf, ha=HA(new_node_ip or node_ha.host, new_node_port or node_ha.port), cliha=HA(new_client_ip or cli_ha.host, new_client_port or cli_ha.port)) looper.add(restartedNode) # replace node in txnPoolNodeSet try: idx = next(i for i, n in enumerate(txnPoolNodeSet) if n.name == node.name) except StopIteration: raise Exception('{} is not the pool'.format(node)) txnPoolNodeSet[idx] = restartedNode looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_ensure_pool_functional(looper, txnPoolNodeSet, steward_wallet, sdk_pool_handle) return restartedNode
def test_replica_removing_before_vc_with_primary_disconnected( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf, tdir, allPluginsPath, chkFreqPatched, view_change): """ 1. Remove replica 2. Reconnect master primary 3. Check that nodes and replicas correctly added """ ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) node = txnPoolNodeSet[0] start_replicas_count = node.replicas.num_replicas instance_id = start_replicas_count - 1 node.replicas.remove_replica(instance_id) _check_replica_removed(node, start_replicas_count, instance_id) assert not node.monitor.isMasterDegraded() assert len(node.requests) == 0 # trigger view change on all nodes disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node) txnPoolNodeSet.remove(node) looper.removeProdable(node) node = start_stopped_node(node, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet.append(node) looper.run(checkNodesConnected(txnPoolNodeSet)) waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=1, customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) assert start_replicas_count == node.replicas.num_replicas
def test_catchup_with_ledger_statuses_in_old_format_from_one_node( txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath): """ A node is restarted and during a catch-up receives ledger statuses in an old format (without `protocolVersion`) from one of nodes in the pool. The test verifies that the node successfully completes the catch-up and participates in ordering of further transactions. """ node_to_restart = txnPoolNodeSet[-1] other_nodes = txnPoolNodeSet[:-1] old_node = txnPoolNodeSet[0] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) original_get_ledger_status = old_node.getLedgerStatus # Patch the method getLedgerStatus to # get_ledger_status_without_protocol_version for sending ledger status # in old format (without `protocolVersion`) def get_ledger_status_without_protocol_version(ledgerId: int): original_ledger_status = original_get_ledger_status(ledgerId) return LedgerStatusInOldFormat(original_ledger_status.ledgerId, original_ledger_status.txnSeqNo, original_ledger_status.viewNo, original_ledger_status.ppSeqNo, original_ledger_status.merkleRoot) old_node.getLedgerStatus = get_ledger_status_without_protocol_version # restart node disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_to_restart) looper.removeProdable(name=node_to_restart.name) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2) # add `node_to_restart` to pool node_to_restart = start_stopped_node(node_to_restart, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[-1] = node_to_restart looper.run(checkNodesConnected(txnPoolNodeSet)) # Verify that `node_to_restart` successfully completes catch-up waitNodeDataEquality(looper, node_to_restart, *other_nodes) # check discarding ledger statuses from `old_node` for all ledgers assert countDiscarded(node_to_restart, 'replied message has invalid structure') >= 3 # Verify that `node_to_restart` participates in ordering # of further transactions sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) waitNodeDataEquality(looper, node_to_restart, *other_nodes)
def testClientConnectsToNewNode(looper, txnPoolNodeSet, tdir, client_tdir, tconf, steward1, stewardWallet, allPluginsPath): """ A client should be able to connect to a newly added node """ newStewardName = "testClientSteward" + randomString(3) newNodeName = "Epsilon" oldNodeReg = copy(steward1.nodeReg) newSteward, newStewardWallet, newNode = addNewStewardAndNode( looper, steward1, stewardWallet, newStewardName, newNodeName, tdir, client_tdir, tconf, allPluginsPath) txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet)) logger.debug("{} connected to the pool".format(newNode)) def chkNodeRegRecvd(): assert (len(steward1.nodeReg) - len(oldNodeReg)) == 1 assert (newNode.name + CLIENT_STACK_SUFFIX) in steward1.nodeReg timeout = waits.expectedClientToPoolConnectionTimeout(len(txnPoolNodeSet)) looper.run(eventually(chkNodeRegRecvd, retryWait=1, timeout=timeout)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def test_get_last_ordered_timestamp_after_catchup(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath): node_to_disconnect = txnPoolNodeSet[-1] reply_before = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1)[0][1] looper.runFor(2) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_to_disconnect) looper.removeProdable(name=node_to_disconnect.name) reply = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1)[0][1] node_to_disconnect = start_stopped_node(node_to_disconnect, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[-1] = node_to_disconnect looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, node_to_disconnect, *txnPoolNodeSet[:-1], exclude_from_check=['check_last_ordered_3pc_backup']) ts_from_state = node_to_disconnect.master_replica._get_last_timestamp_from_state( DOMAIN_LEDGER_ID) assert ts_from_state == get_txn_time(reply['result']) assert ts_from_state != get_txn_time(reply_before['result'])
def testNodeCatchupAfterRestart(newNodeCaughtUp, txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns): """ A node that restarts after some transactions should eventually get the transactions which happened while it was down :return: """ looper, newNode, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns logger.debug("Stopping node {} with pool ledger size {}".format( newNode, newNode.poolManager.txnSeqNo)) ensureNodeDisconnectedFromPool(looper, txnPoolNodeSet, newNode) # for n in txnPoolNodeSet[:4]: # for r in n.nodestack.remotes.values(): # if r.name == newNode.name: # r.removeStaleCorrespondents() # looper.run(eventually(checkNodeDisconnectedFrom, newNode.name, # txnPoolNodeSet[:4], retryWait=1, timeout=5)) # TODO: Check if the node has really stopped processing requests? logger.debug("Sending requests") sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) logger.debug("Starting the stopped node, {}".format(newNode)) newNode.start(looper.loop) looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run( eventually(checkNodeLedgersForEquality, newNode, *txnPoolNodeSet[:4], retryWait=1, timeout=15))
def add_started_node(looper, new_node, txnPoolNodeSet, client_tdir, stewardClient, stewardWallet, sigseed, bls_key): ''' Adds already created node to the pool, that is sends NODE txn. Makes sure that node is actually added and connected to all otehr nodes. ''' newSteward, newStewardWallet = addNewSteward(looper, client_tdir, stewardClient, stewardWallet, "Steward" + new_node.name, clientClass=TestClient) node_name = new_node.name send_new_node_txn(sigseed, new_node.poolManager.nodeReg[node_name][0], new_node.poolManager.nodeReg[node_name][1], new_node.poolManager.cliNodeReg[node_name + "C"][0], new_node.poolManager.cliNodeReg[node_name + "C"][1], bls_key, node_name, newSteward, newStewardWallet) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
def test_current_state_propagation(newNodeCaughtUp, txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns, tconf, tdir, allPluginsPath): """ Checks that nodes send CurrentState to lagged nodes. """ # 1. Start pool looper, new_node, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns # 2. Stop one node disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node, stopNode=True) looper.removeProdable(new_node) # 3. Start it again restarted_node = start_stopped_node(new_node, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[-1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) looper.runFor(5) # 4. Check that all nodes sent CurrentState for node in txnPoolNodeSet[:-1]: sent_times = node.spylog.count( node.send_current_state_to_lagging_node.__name__) assert sent_times != 0, "{} haven't sent CurrentState".format(node) looper.runFor(5) # 5. Check that it received CurrentState messages received_times = restarted_node.spylog.count( restarted_node.process_current_state_message.__name__) assert received_times != 0
def testClientConnectsToNewNode(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, allPluginsPath): """ A client should be able to connect to a newly added node """ newStewardName = "testClientSteward"+randomString(3) newNodeName = "Epsilon" oldNodeReg = copy(steward1.nodeReg) newSteward, newStewardWallet, newNode = addNewStewardAndNode(looper, steward1, stewardWallet, newStewardName, newNodeName, tdirWithPoolTxns, tconf, allPluginsPath) txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet)) logger.debug("{} connected to the pool".format(newNode)) def chkNodeRegRecvd(): assert (len(steward1.nodeReg) - len(oldNodeReg)) == 1 assert (newNode.name + CLIENT_STACK_SUFFIX) in steward1.nodeReg looper.run(eventually(chkNodeRegRecvd, retryWait=1, timeout=5)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def testAddInactiveNodeThenActivate(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle, tdir, tconf, allPluginsPath): new_steward_name = "testClientSteward" + randomString(3) new_node_name = "Kappa" # adding a new node without SERVICES field # it means the node is in the inactive state new_steward_wallet, new_node = \ sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, new_steward_name, new_node_name, tdir, tconf, allPluginsPath, services=None) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) new_node = update_node_data_and_reconnect(looper, txnPoolNodeSet + [new_node], new_steward_wallet, sdk_pool_handle, new_node, None, None, None, None, tdir, tconf) txnPoolNodeSet.append(new_node) sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet, sdk_pool_handle)
def test_state_regenerated_from_ledger(looper, nodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath): """ Node loses its state database but recreates it from ledger after start. Checking ATTRIB txns too since they store some data off ledger too """ trust_anchors = [] for i in range(5): trust_anchors.append(sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'TA' + str(i), TRUST_ANCHOR_STRING)) sdk_add_raw_attribute(looper, sdk_pool_handle, trust_anchors[-1], randomString(6), randomString(10)) for wh in trust_anchors: for i in range(3): sdk_add_new_nym(looper, sdk_pool_handle, wh, 'NP1' + str(i)) ensure_all_nodes_have_same_data(looper, nodeSet) node_to_stop = nodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv.db_path node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, nodeSet[:-1]) shutil.rmtree(state_db_path) config_helper = NodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode( node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) nodeSet[-1] = restarted_node looper.run(checkNodesConnected(nodeSet)) # Need some time as `last_ordered_3PC` is compared too and that is # communicated through catchup waitNodeDataEquality(looper, restarted_node, *nodeSet[:-1]) # Pool is still functional for wh in trust_anchors: sdk_add_new_nym(looper, sdk_pool_handle, wh, 'NP--' + randomString(5)) ensure_all_nodes_have_same_data(looper, nodeSet)
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes): """ Case 2 - A node making nominations for a multiple other nodes. Consider 4 nodes A, B, C, and D. Lets say node B is malicious and nominates node C to all nodes. Again node B nominates node D to all nodes. """ nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() looper.run(checkNodesConnected(nodeSet)) # Node B sends multiple NOMINATE msgs but only after A has nominated itself looper.run(eventually(checkNomination, A, A.name, retryWait=.25, timeout=1)) instId = getSelfNominationByNode(A) BRep = Replica.generateName(B.name, instId) CRep = Replica.generateName(C.name, instId) DRep = Replica.generateName(D.name, instId) # Node B first sends NOMINATE msgs for Node C to all nodes B.send(Nomination(CRep, instId, B.viewNo)) # Node B sends NOMINATE msgs for Node D to all nodes B.send(Nomination(DRep, instId, B.viewNo)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45) # All nodes from node A, node C, node D(node B is malicious anyway so # not considering it) should have nomination for node C from node B since # node B first nominated node C for node in [A, C, D]: assert node.elector.nominations[instId][BRep] == CRep
def test_restarted_node_catches_up_config_ledger_txns( looper, some_config_txns_done, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, sdk_new_node_caught_up, keys, tconf, tdir, allPluginsPath): """ A node is stopped, a few config ledger txns happen, the stopped node is started and catches up the config ledger """ new_node = sdk_new_node_caught_up disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node, stopNode=True) looper.removeProdable(new_node) # Do some config txns; using a fixture as a method, passing some arguments # as None as they only make sense for the fixture (pre-requisites) send_some_config_txns(looper, sdk_pool_handle, sdk_wallet_client, keys) # Make sure new node got out of sync for node in txnPoolNodeSet[:-1]: assert new_node.configLedger.size < node.configLedger.size restarted_node = start_stopped_node(new_node, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[-1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, restarted_node, *txnPoolNodeSet[:-1], exclude_from_check=['check_last_ordered_3pc_backup'])
def testAdd2NewNodes(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, allPluginsPath): """ Add 2 new nodes to trigger replica addition and primary election """ for nodeName in ("Zeta", "Eta"): newStewardName = "testClientSteward"+randomString(3) newSteward, newStewardWallet, newNode = addNewStewardAndNode(looper, steward1, stewardWallet, newStewardName, nodeName, tdirWithPoolTxns, tconf, allPluginsPath) txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet)) logger.debug("{} connected to the pool".format(newNode)) looper.run(eventually(checkNodeLedgersForEquality, newNode, *txnPoolNodeSet[:-1], retryWait=1, timeout=7)) f = getMaxFailures(len(txnPoolNodeSet)) def checkFValue(): for node in txnPoolNodeSet: assert node.f == f assert len(node.replicas) == (f + 1) looper.run(eventually(checkFValue, retryWait=1, timeout=5)) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1, timeout=5)
def test_propagate_primary_after_primary_restart_view_0( looper, txnPoolNodeSet, tconf, sdk_pool_handle, sdk_wallet_steward, tdir, allPluginsPath): """ Delay instance change msgs to prevent view change during primary restart to test propagate primary for primary node. ppSeqNo should be > 0 to be able to check that propagate primary restores all indices correctly case viewNo == 0 """ sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) old_ppseqno = _get_ppseqno(txnPoolNodeSet) assert (old_ppseqno > 0) old_viewNo = checkViewNoForNodes(txnPoolNodeSet) old_primary = get_master_primary_node(txnPoolNodeSet) delay_instance_change(txnPoolNodeSet, IC_DELAY_SEC) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, old_primary, stopNode=True) looper.removeProdable(old_primary) logger.info("Restart node {}".format(old_primary)) restartedNode = start_stopped_node(old_primary, looper, tconf, tdir, allPluginsPath, delay_instance_change_msgs=False) idx = [ i for i, n in enumerate(txnPoolNodeSet) if n.name == restartedNode.name ][0] txnPoolNodeSet[idx] = restartedNode restartedNode.nodeIbStasher.delay(icDelay(IC_DELAY_SEC)) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) new_viewNo = checkViewNoForNodes(txnPoolNodeSet) assert (new_viewNo == old_viewNo) new_primary = get_master_primary_node(txnPoolNodeSet) assert (new_primary.name == old_primary.name) # check ppSeqNo the same _get_ppseqno(txnPoolNodeSet) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) new_ppseqno = _get_ppseqno(txnPoolNodeSet) assert (new_ppseqno > old_ppseqno)
def test_removed_replica_restored_on_view_change( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf, tdir, allPluginsPath, chkFreqPatched, view_change): """ 1. Remove replica on some node which is not master primary 2. Reconnect the node which was master primary so far 3. Check that nodes and replicas correctly added """ ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) node = get_last_master_non_primary_node(txnPoolNodeSet) start_replicas_count = node.replicas.num_replicas instance_id = start_replicas_count - 1 node.replicas.remove_replica(instance_id) check_replica_removed(node, start_replicas_count, instance_id) # trigger view change on all nodes master_primary = get_master_primary_node(txnPoolNodeSet) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, master_primary) txnPoolNodeSet.remove(master_primary) looper.removeProdable(master_primary) looper.runFor(tconf.ToleratePrimaryDisconnection + 2) restarted_node = start_stopped_node(master_primary, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet.append(restarted_node) looper.run(checkNodesConnected(txnPoolNodeSet)) waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=1, customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) assert start_replicas_count == node.replicas.num_replicas
def testReplayLedger(addNymTxn, addedRawAttribute, submittedPublicKeys, nodeSet, looper, tconf, tdirWithPoolTxns, allPluginsPath, txnPoolNodeSet): """ stop first node (which will clean graph db too) then restart node """ nodeToStop = nodeSet[0] nodeToStop.cleanupOnStopping = False nodeToStop.stop() looper.removeProdable(nodeToStop) #client = nodeToStop.graphStore.client #client.db_drop(client._connection.db_opened) newNode = TestNode(nodeToStop.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=nodeToStop.nodestack.ha, cliha=nodeToStop.clientstack.ha) looper.add(newNode) nodeSet[0] = newNode looper.run(checkNodesConnected(nodeSet, overrideTimeout=30)) looper.run( eventually(checkNodeLedgersForEquality, newNode, *txnPoolNodeSet[1:4], retryWait=1, timeout=15)) compareGraph("NYM", nodeSet) compareGraph("IssuerKey", nodeSet) compareGraph("Schema", nodeSet)
def test_fill_ts_store_after_catchup(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath): sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) node_to_disconnect = txnPoolNodeSet[-1] disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_to_disconnect) looper.removeProdable(name=node_to_disconnect.name) sdk_replies = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2) node_to_disconnect = start_stopped_node(node_to_disconnect, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[-1] = node_to_disconnect looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, node_to_disconnect, *txnPoolNodeSet, exclude_from_check=['check_last_ordered_3pc_backup']) req_handler = node_to_disconnect.read_manager.request_handlers[GET_BUY] for reply in sdk_replies: key = BuyHandler.prepare_buy_key(get_from(reply[1]['result']), get_req_id(reply[1]['result'])) root_hash = req_handler.database_manager.ts_store.get_equal_or_prev( get_txn_time(reply[1]['result'])) assert root_hash from_state = req_handler.state.get_for_root_hash(root_hash=root_hash, key=key) assert domain_state_serializer.deserialize(from_state)['amount'] == \ get_payload_data(reply[1]['result'])['amount']
def sdk_add_2_nodes(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, allPluginsPath): names = ("Zeta", "Eta") new_nodes = [] for node_name in names: new_steward_name = "testClientSteward" + randomString(3) new_steward_wallet, new_node = \ sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, new_steward_name, node_name, tdir, tconf, allPluginsPath) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality( looper, new_node, *txnPoolNodeSet[:-1], exclude_from_check=['check_last_ordered_3pc_backup']) sdk_pool_refresh(looper, sdk_pool_handle) new_nodes.append(new_node) return new_nodes
def test_upper_bound_of_checkpoint_after_catchup_is_divisible_by_chk_freq( chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, sdk_wallet_client, tdir, tconf, allPluginsPath): sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 4) _, new_node = sdk_add_new_steward_and_node( looper, sdk_pool_handle, sdk_wallet_steward, 'EpsilonSteward', 'Epsilon', tdir, tconf, allPluginsPath=allPluginsPath) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1], exclude_from_check=['check_last_ordered_3pc_backup']) # Epsilon did not participate in ordering of the batch with EpsilonSteward # NYM transaction and the batch with Epsilon NODE transaction. # Epsilon got these transactions via catch-up. sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) for replica in new_node.replicas.values(): assert len(replica.checkpoints) == 1 assert next(iter(replica.checkpoints)) == (7, 10)
def test_number_txns_in_catchup_and_vc_queue_valid(looper, txnPoolNodeSet, tconf, sdk_pool_handle, sdk_wallet_steward, tdir, allPluginsPath): num_txns = 5 master_node = get_master_primary_node(txnPoolNodeSet) master_node_index = txnPoolNodeSet.index(master_node) other_nodes = txnPoolNodeSet.copy() other_nodes.remove(master_node) old_view = master_node.viewNo expected_view_no = old_view + 1 disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, master_node, stopNode=True) looper.removeProdable(master_node) looper.run(eventually(checkViewNoForNodes, other_nodes, expected_view_no, retryWait=1, timeout=tconf.NEW_VIEW_TIMEOUT)) sdk_pool_refresh(looper, sdk_pool_handle) sdk_send_random_and_check(looper, other_nodes, sdk_pool_handle, sdk_wallet_steward, num_txns) master_node = start_stopped_node(master_node, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[master_node_index] = master_node looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, master_node, *txnPoolNodeSet[-1:], exclude_from_check=['check_last_ordered_3pc_backup']) latest_info = master_node._info_tool.info assert latest_info['Node_info']['Catchup_status']['Number_txns_in_catchup'][1] == num_txns assert latest_info['Node_info']['View_change_status']['View_No'] == expected_view_no for n in other_nodes: assert n._info_tool.info['Node_info']['View_change_status']['Last_complete_view_no'] == expected_view_no
def testNodeSchedulesUpgradeAfterRestart(upgradeScheduled, looper, nodeSet, validUpgrade, testNodeClass, tdirWithPoolTxns, tconf, allPluginsPath): names = [] while nodeSet: node = nodeSet.pop() names.append(node.name) node.cleanupOnStopping = False looper.removeProdable(node) node.stop() del node for nm in names: node = testNodeClass(nm, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath) looper.add(node) nodeSet.append(node) looper.run(checkNodesConnected(nodeSet)) ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=10) looper.run( eventually(checkUpgradeScheduled, nodeSet, validUpgrade[VERSION], retryWait=1, timeout=10))
def add_new_node(helpers, looper, node_set, sdk_wallet_steward, current_amount, seq_no, fees_set, sdk_pool_handle, tdir, tconf, allPluginsPath, address, do_post_node_creation): new_did, verkey = helpers.wallet.create_did(sdk_wallet=sdk_wallet_steward) req = helpers.request.nym(sdk_wallet=sdk_wallet_steward, alias="new_steward", role=STEWARD_STRING, dest=new_did, verkey=verkey) utxos = [{ADDRESS: address, AMOUNT: current_amount, f.SEQ_NO.nm: seq_no}] req = add_fees_request_with_address(helpers, fees_set, req, address, utxos=utxos) current_amount, seq_no, _ = send_and_check_nym_with_fees(helpers, fees_set, seq_no, looper, addresses, current_amount, nym_with_fees=req) new_steward_wallet_handle = sdk_wallet_steward[0], new_did new_node = sdk_add_new_node(looper, sdk_pool_handle, new_steward_wallet_handle, 'Epsilon', tdir, tconf, allPluginsPath=allPluginsPath, do_post_node_creation=do_post_node_creation) node_set.append(new_node) looper.run(checkNodesConnected(node_set)) waitNodeDataEquality(looper, new_node, *node_set[:-1])
def testNodePortChanged(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, nodeThetaAdded): """ An running node's port is changed """ newSteward, newStewardWallet, newNode = nodeThetaAdded nodeNewHa, clientNewHa = genHa(2) logger.debug("{} changing HAs to {} {}".format(newNode, nodeNewHa, clientNewHa)) changeNodeHa(looper, newSteward, newStewardWallet, newNode, nodeHa=nodeNewHa, clientHa=clientNewHa) newNode.stop() looper.removeProdable(name=newNode.name) logger.debug("{} starting with HAs {} {}".format(newNode, nodeNewHa, clientNewHa)) node = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, config=tconf, ha=nodeNewHa, cliha=clientNewHa) looper.add(node) # The last element of `txnPoolNodeSet` is the node Theta that was just # stopped txnPoolNodeSet[-1] = node looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run(eventually(checkNodeLedgersForEquality, node, *txnPoolNodeSet[:-1], retryWait=1, timeout=10)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet)
def nodeThetaAdded(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, allPluginsPath, testNodeClass=None, testClientClass=None, name=None): newStewardName = "testClientSteward" + randomString(3) newNodeName = name or "Theta" newSteward, newStewardWallet, newNode = addNewStewardAndNode( looper, steward1, stewardWallet, newStewardName, newNodeName, tdirWithPoolTxns, tconf, allPluginsPath, nodeClass=testNodeClass, clientClass=testClientClass) txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet) return newSteward, newStewardWallet, newNode
def testNodeRejectingInvalidTxns(conf, txnPoolNodeSet, patched_node, nodeCreatedAfterSomeTxns): """ A newly joined node is catching up and sends catchup requests to other nodes but one of the nodes replies with incorrect transactions. The newly joined node detects that and rejects the transactions and thus blacklists the node. Ii thus cannot complete the process till the timeout and then requests the missing transactions. """ looper, newNode, client, wallet, _, _ = nodeCreatedAfterSomeTxns bad_node = patched_node do_not_tell_clients_about_newly_joined_node(txnPoolNodeSet) logger.debug('Catchup request processor of {} patched'.format(bad_node)) looper.run(checkNodesConnected(txnPoolNodeSet)) # catchup #1 -> CatchupTransactionsTimeout -> catchup #2 catchup_timeout = waits.expectedPoolCatchupTime(len(txnPoolNodeSet) + 1) timeout = 2 * catchup_timeout + conf.CatchupTransactionsTimeout # have to skip seqno_db check because the txns are not executed # on the new node waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1], customTimeout=timeout) assert newNode.isNodeBlacklisted(bad_node.name)
def setupNodesAndClient(looper: Looper, nodes: Sequence[TestNode], nodeReg=None, tmpdir=None): looper.run(checkNodesConnected(nodes)) ensureElectionsDone(looper=looper, nodes=nodes) return setupClient(looper, nodes, nodeReg=nodeReg, tmpdir=tmpdir)
def test_integration_setup_last_ordered_after_catchup(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_wallet_client, sdk_pool_handle, tdir, tconf, allPluginsPath): sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) _, new_node = sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, 'EpsilonSteward', 'Epsilon', tdir, tconf, allPluginsPath=allPluginsPath) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1], exclude_from_check=['check_last_ordered_3pc_backup']) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) looper.run(eventually(replicas_synced, new_node)) for node in txnPoolNodeSet: for replica in node.replicas.values(): assert replica.last_ordered_3pc == (0, 4) if not replica.isMaster: assert get_count(replica, replica._request_three_phase_msg) == 0
def testAddInactiveNodeThenActivate(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, steward1, stewardWallet, allPluginsPath): newStewardName = "testClientSteward" + randomString(3) newNodeName = "Kappa" # adding a new node without SERVICES field # it means the node is in the inactive state def del_services(op): del op[DATA][SERVICES] newSteward, newStewardWallet, newNode = \ addNewStewardAndNode(looper, steward1, stewardWallet, newStewardName, newNodeName, tdirWithPoolTxns, tconf, allPluginsPath, transformNodeOpFunc=del_services) looper.run(checkNodesConnected(txnPoolNodeSet)) # turn the new node on node_data = {ALIAS: newNode.name, SERVICES: [VALIDATOR]} updateNodeDataAndReconnect(looper, newSteward, newStewardWallet, newNode, node_data, tdirWithPoolTxns, tconf, txnPoolNodeSet + [newNode])
def sdk_node_theta_added(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_steward, allPluginsPath, testNodeClass=None, name=None): new_steward_name = "testClientSteward" + randomString(3) new_node_name = name or "Theta" new_steward_wallet, new_node = \ sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, new_steward_name, new_node_name, tdir, tconf, allPluginsPath, nodeClass=testNodeClass) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) return new_steward_wallet, new_node
def test_order_after_demote_and_restart(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tdir, tconf, allPluginsPath, sdk_wallet_stewards): sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 3, 3) primary_node = txnPoolNodeSet[0] node_to_stop = txnPoolNodeSet[1] node_to_demote = txnPoolNodeSet[2] txnPoolNodeSet.remove(node_to_demote) node_to_stop.cleanupOnStopping = True node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, txnPoolNodeSet, timeout=2) demote_node(looper, sdk_wallet_stewards[2], sdk_pool_handle, node_to_demote) config_helper = PNodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode(node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) txnPoolNodeSet[1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1, 1) def get_current_bls_keys(node): return node.master_replica._bls_bft_replica._bls_bft.bls_key_register._current_bls_keys assert get_current_bls_keys(restarted_node) == get_current_bls_keys(primary_node)
def add_new_node(looper, nodes, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, all_plugins_path, name=None): node_name = name or "Psi" new_steward_name = "testClientSteward" + randomString(3) _, new_node = sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, new_steward_name, node_name, tdir, tconf, allPluginsPath=all_plugins_path) nodes.append(new_node) looper.run(checkNodesConnected(nodes)) timeout = waits.expectedPoolCatchupTime(nodeCount=len(nodes)) waitNodeDataEquality(looper, new_node, *nodes[:-1], customTimeout=timeout, exclude_from_check=['check_last_ordered_3pc_backup']) sdk_pool_refresh(looper, sdk_pool_handle) return new_node
def test_cancel_request_cp_and_ls_after_catchup(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath): '''Test cancel of schedule with requesting ledger statuses and consistency proofs after catchup.''' node_to_disconnect = txnPoolNodeSet[-1] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) # restart node disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_to_disconnect) looper.removeProdable(name=node_to_disconnect.name) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2) # add node_to_disconnect to pool node_to_disconnect = start_stopped_node(node_to_disconnect, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[-1] = node_to_disconnect looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, node_to_disconnect, *txnPoolNodeSet) # check cancel of schedule with requesting ledger statuses and consistency proofs for event in node_to_disconnect.timer._events: name = event.callback.__name__ assert name != '_reask_for_ledger_status' assert name != '_reask_for_last_consistency_proof'
def sdk_node_theta_added(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_steward, allPluginsPath, testNodeClass=TestNode, name=None): new_steward_name = "testClientSteward" + randomString(3) new_node_name = name or "Theta" new_steward_wallet, new_node = \ sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, new_steward_name, new_node_name, tdir, tconf, allPluginsPath, nodeClass=testNodeClass) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) return new_steward_wallet, new_node
def updateNodeDataAndReconnect(looper, steward, stewardWallet, node, node_data, tdirWithPoolTxns, tconf, txnPoolNodeSet): updateNodeData(looper, steward, stewardWallet, node, node_data) # restart the Node with new HA node.stop() node_alias = node_data.get(ALIAS, None) or node.name node_ip = node_data.get(NODE_IP, None) or node.nodestack.ha.host node_port = node_data.get(NODE_PORT, None) or node.nodestack.ha.port client_ip = node_data.get(CLIENT_IP, None) or node.clientstack.ha.host client_port = node_data.get(CLIENT_PORT, None) or node.clientstack.ha.port looper.removeProdable(name=node.name) restartedNode = TestNode(node_alias, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, config=tconf, ha=HA(node_ip, node_port), cliha=HA(client_ip, client_port)) looper.add(restartedNode) # replace node in txnPoolNodeSet try: idx = next(i for i, n in enumerate(txnPoolNodeSet) if n.name == node.name) except StopIteration: raise Exception('{} is not the pool'.format(node)) txnPoolNodeSet[idx] = restartedNode looper.run(checkNodesConnected(txnPoolNodeSet)) return restartedNode
def testNodeRemoveUnknownRemote(allPluginsPath, tdir_for_func, tconf_for_func, looper, txnPoolNodeSetNotStarted): """ The nodes Alpha and Beta know about each other so they should connect but they should remove remote for C when it tries to connect to them """ nodes = txnPoolNodeSetNotStarted[:2] for node in nodes: tellKeysToOthers(node, nodes) A, B = nodes for node in nodes: looper.add(node) looper.run(checkNodesConnected(nodes)) C = txnPoolNodeSetNotStarted[2] for node in nodes: tellKeysToOthers(node, [C, ]) looper.add(C) looper.runFor(5) stopNodes([C, ], looper) stopNodes([A, B], looper) for node in [A, B, C]: looper.removeProdable(node)
def testNodeKeysChanged(looper, txnPoolNodeSet, tdir, tconf, sdk_node_theta_added, sdk_pool_handle, allPluginsPath=None): new_steward_wallet, new_node = sdk_node_theta_added new_node.stop() looper.removeProdable(name=new_node.name) nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha) sigseed = randomString(32).encode() verkey = base58.b58encode(SimpleSigner(seed=sigseed).naclSigner.verraw).decode("utf-8") sdk_change_node_keys(looper, new_node, new_steward_wallet, sdk_pool_handle, verkey) config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir) initNodeKeysForBothStacks(new_node.name, config_helper.keys_dir, sigseed, override=True) logger.debug("{} starting with HAs {} {}".format(new_node, nodeHa, nodeCHa)) node = TestNode(new_node.name, config_helper=config_helper, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) looper.add(node) # The last element of `txnPoolNodeSet` is the node Theta that was just # stopped txnPoolNodeSet[-1] = node looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet, sdk_pool_handle)
def test_restarted_node_catches_up_config_ledger_txns(looper, some_config_txns_done, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, sdk_new_node_caught_up, keys, tconf, tdir, allPluginsPath): """ A node is stopped, a few config ledger txns happen, the stopped node is started and catches up the config ledger """ new_node = sdk_new_node_caught_up disconnect_node_and_ensure_disconnected( looper, txnPoolNodeSet, new_node, stopNode=True) looper.removeProdable(new_node) # Do some config txns; using a fixture as a method, passing some arguments # as None as they only make sense for the fixture (pre-requisites) send_some_config_txns(looper, sdk_pool_handle, sdk_wallet_client, keys) # Make sure new node got out of sync for node in txnPoolNodeSet[:-1]: assert new_node.configLedger.size < node.configLedger.size restarted_node = start_stopped_node(new_node, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[-1] = restarted_node looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, restarted_node, *txnPoolNodeSet[:-1])
def add_started_node(looper, new_node, node_ha, client_ha, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, bls_key, key_proof): ''' Adds already created node to the pool, that is sends NODE txn. Makes sure that node is actually added and connected to all otehr nodes. ''' new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, "Steward" + new_node.name, role=STEWARD_STRING) node_name = new_node.name node_dest = hexToFriendly(new_node.nodestack.verhex) sdk_send_update_node(looper, new_steward_wallet_handle, sdk_pool_handle, node_dest, node_name, node_ha[0], node_ha[1], client_ha[0], client_ha[1], services=[VALIDATOR], bls_key=bls_key, key_proof=key_proof) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes): """ Case 5 - A node making primary declarations for a multiple other nodes. Consider 4 nodes A, B, C, and D. Lets say node B is malicious and declares node C as primary to all nodes. Again node B declares node D as primary to all nodes. """ nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() looper.run(checkNodesConnected(nodeSet)) BRep = Replica.generateName(B.name, 0) CRep = Replica.generateName(C.name, 0) DRep = Replica.generateName(D.name, 0) # Node B first sends PRIMARY msgs for Node C to all nodes B.send(Primary(CRep, 0, B.viewNo)) # Node B sends PRIMARY msgs for Node D to all nodes B.send(Primary(DRep, 0, B.viewNo)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45) # All nodes from node A, node C, node D(node B is malicious anyway so not # considering it) should have primary declarations for node C from node B # since node B first nominated node C for node in [A, C, D]: logger.debug("node {} should have primary declaration for C from node B".format(node)) assert node.elector.primaryDeclarations[0][BRep] == CRep
def test_state_regenerated_from_ledger(looper, tdirWithPoolTxns, tdirWithDomainTxnsUpdated, nodeSet, tconf, trustee, trusteeWallet, allPluginsPath): """ Node loses its state database but recreates it from ledger after start. Checking ATTRIB txns too since they store some data off ledger too """ trust_anchors = [] for i in range(5): trust_anchors.append(getClientAddedWithRole(nodeSet, tdirWithPoolTxns, looper, trustee, trusteeWallet, 'TA' + str(i), role=TRUST_ANCHOR)) addRawAttribute(looper, *trust_anchors[-1], randomString(6), randomString(10), dest=trust_anchors[-1][1].defaultId) for tc, tw in trust_anchors: for i in range(3): getClientAddedWithRole(nodeSet, tdirWithPoolTxns, looper, tc, tw, 'NP1' + str(i)) ensure_all_nodes_have_same_data(looper, nodeSet) node_to_stop = nodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv._dbPath node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop.name, nodeSet[:-1]) shutil.rmtree(state_db_path) restarted_node = TestNode(node_to_stop.name, basedirpath=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) nodeSet[-1] = restarted_node looper.run(checkNodesConnected(nodeSet)) # Need some time as `last_ordered_3PC` is compared too and that is # communicated through catchup waitNodeDataEquality(looper, restarted_node, *nodeSet[:-1]) # Pool is still functional for tc, tw in trust_anchors: getClientAddedWithRole(nodeSet, tdirWithPoolTxns, looper, tc, tw, 'NP--{}'.format(tc.name)) ensure_all_nodes_have_same_data(looper, nodeSet)
def elections_done(case_6_setup, looper, txnPoolNodeSet): # Make sure elections are done successfully A, B, C, D = txnPoolNodeSet looper.run(checkNodesConnected(txnPoolNodeSet)) inst_ids = (0, 1) def chk(): # Check that each Primary is received by A before A has sent any # Primary primary_recv_times = { i: [entry.starttime for entry in A.elector.spylog.getAll( A.elector.processPrimary) if entry.params['prim'].instId == i] for i in inst_ids } primary_send_times = { i: [entry.starttime for entry in A.elector.spylog.getAll( A.elector.sendPrimary) if entry.params['instId'] == 0] for i in inst_ids } for i in inst_ids: assert primary_send_times[i][0] > max(primary_recv_times[i]) looper.run(eventually(chk, retryWait=1, timeout=15)) checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1) # Make sure no Nominations or Primary are received by A from B for i in inst_ids: assert B.replicas[i].name not in A.elector.nominations[i] assert B.replicas[i].name not in A.elector.primaryDeclarations[i]
def test_get_last_ordered_timestamp_after_catchup(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath): node_to_disconnect = txnPoolNodeSet[-1] reply_before = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1)[0][1] looper.runFor(2) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_to_disconnect) looper.removeProdable(name=node_to_disconnect.name) reply = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1)[0][1] node_to_disconnect = start_stopped_node(node_to_disconnect, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet[-1] = node_to_disconnect looper.run(checkNodesConnected(txnPoolNodeSet)) waitNodeDataEquality(looper, node_to_disconnect, *txnPoolNodeSet[:-1]) ts_from_state = node_to_disconnect.master_replica._get_last_timestamp_from_state(DOMAIN_LEDGER_ID) assert ts_from_state == get_txn_time(reply['result']) assert ts_from_state != get_txn_time(reply_before['result'])
def testPrimaryElectionCase4(case4Setup, looper): """ Case 4 - A node making multiple primary declarations for a particular node. Consider 4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly declaring Node D as primary """ allNodes = case4Setup A, B, C, D = allNodes looper.run(checkNodesConnected(allNodes)) # Node B sends multiple declarations of node D's 0th protocol instance as # primary to all nodes for i in range(5): B.send(Primary(D.name, 0, B.viewNo)) # No node from node A, node C, node D(node B is malicious anyway so not # considering it) should have more than one primary declaration for node # D since node D is slow. The one primary declaration for node D, # that nodes A, C and D might have would be because of node B def x(): primDecs = list(node.elector.primaryDeclarations[0].values()) assert primDecs.count(D.name) <= 1 for node in (A, C, D): looper.run(eventually(x, retryWait=.5, timeout=2)) ensureElectionsDone(looper=looper, nodes=allNodes, retryWait=1, timeout=45) # Node D should not have any primary replica assert not D.hasPrimary
def nodeSetWithNodeAddedAfterSomeTxns(txnPoolNodeSet, nodeCreatedAfterSomeTxns): looper, newNode, client, wallet, newStewardClient, newStewardWallet = nodeCreatedAfterSomeTxns txnPoolNodeSet.append(newNode) looper.run(checkNodesConnected(txnPoolNodeSet, overrideTimeout=10)) looper.run(newStewardClient.ensureConnectedToNodes()) looper.run(client.ensureConnectedToNodes()) return looper, newNode, client, wallet, newStewardClient, newStewardWallet
def testNodeCatchupAfterRestart(newNodeCaughtUp, txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns): """ A node that restarts after some transactions should eventually get the transactions which happened while it was down :return: """ looper, newNode, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns logger.debug("Stopping node {} with pool ledger size {}". format(newNode, newNode.poolManager.txnSeqNo)) ensureNodeDisconnectedFromPool(looper, txnPoolNodeSet, newNode) # for n in txnPoolNodeSet[:4]: # for r in n.nodestack.remotes.values(): # if r.name == newNode.name: # r.removeStaleCorrespondents() # looper.run(eventually(checkNodeDisconnectedFrom, newNode.name, # txnPoolNodeSet[:4], retryWait=1, timeout=5)) # TODO: Check if the node has really stopped processing requests? logger.debug("Sending requests") sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) logger.debug("Starting the stopped node, {}".format(newNode)) newNode.start(looper.loop) looper.run(checkNodesConnected(txnPoolNodeSet)) looper.run(eventually(checkNodeLedgersForEquality, newNode, *txnPoolNodeSet[:4], retryWait=1, timeout=15))
def sdk_node_set_with_node_added_after_some_txns( txnPoolNodeSet, sdk_node_created_after_some_txns): looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ sdk_node_created_after_some_txns txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) return looper, new_node, sdk_pool_handle, new_steward_wallet_handle
def test_steward_suspends_node_and_promote_with_new_ha( looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_steward, sdk_node_theta_added, poolTxnStewardData, allPluginsPath): new_steward_wallet, new_node = sdk_node_theta_added looper.run(checkNodesConnected(txnPoolNodeSet + [new_node])) demote_node(looper, new_steward_wallet, sdk_pool_handle, new_node) # Check suspended node does not exist in any nodeReg or remotes of # nodes or clients txnPoolNodeSet = txnPoolNodeSet[:-1] for node in txnPoolNodeSet: looper.run(eventually(checkNodeNotInNodeReg, node, new_node.name)) # Check that a node does not connect to the suspended # node sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet, sdk_pool_handle) with pytest.raises(RemoteNotFound): looper.loop.run_until_complete(sendMessageAndCheckDelivery(txnPoolNodeSet[0], new_node)) new_node.stop() looper.removeProdable(new_node) # Check that a node whose suspension is revoked can reconnect to other # nodes and clients can also connect to that node node_ha, client_ha = genHa(2) node_nym = hexToFriendly(new_node.nodestack.verhex) sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, node_nym, new_node.name, node_ha.host, node_ha.port, client_ha.host, client_ha.port, services=[VALIDATOR]) new_node.nodestack.ha = node_ha new_node.clientstack.ha = client_ha nodeTheta = start_stopped_node(new_node, looper, tconf, tdir, allPluginsPath, delay_instance_change_msgs=False) assert all(node.nodestack.remotes[new_node.name].ha == node_ha for node in txnPoolNodeSet) txnPoolNodeSet.append(nodeTheta) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def testKeyShareParty(tdir_for_func): """ connections to all nodes should be successfully established when key sharing is enabled. """ nodeReg = genNodeReg(5) logger.debug("-----sharing keys-----") with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: for n in nodeSet: n.startKeySharing() looper.run(checkNodesConnected(nodeSet)) logger.debug("-----key sharing done, connect after key sharing-----") with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as loop: loop.run(checkNodesConnected(nodeSet), msgAll(nodeSet))