def testStewardSuspensionByTrustee(looper, sdk_pool_handle, another_trustee, another_steward): _, did_stew = another_steward sdk_suspend_role(looper, sdk_pool_handle, another_trustee, did_stew) with pytest.raises(RequestRejectedException): sdk_add_new_nym(looper, sdk_pool_handle, another_steward, role=TRUST_ANCHOR_STRING)
def test_requests_post_multiple_new_nodes( looper, nodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath, some_transactions_done): new_nodes = [] for node_name in ('Zeta', 'Eta'): new_steward_wallet, new_node = sdk_node_theta_added(looper, nodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath, node_config_helper_class=NodeConfigHelper, testNodeClass=TestNode, name=node_name) new_nodes.append(new_node) for _ in range(5): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee) for new_node in new_nodes: waitNodeDataEquality(looper, new_node, *nodeSet[:-2]) for _ in range(5): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee)
def testTrusteeSuspendingTrustAnchor(looper, sdk_pool_handle, sdk_wallet_trustee, sdk_wallet_trust_anchor): _, did = sdk_wallet_trust_anchor sdk_suspend_role(looper, sdk_pool_handle, sdk_wallet_trustee, did) with pytest.raises(RequestRejectedException) as e: sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trust_anchor) e.match('None role cannot add None role')
def sdk_change_bls_key(looper, txnPoolNodeSet, node, sdk_pool_handle, sdk_wallet_steward, add_wrong=False, new_bls=None): new_blspk = init_bls_keys(node.keys_dir, node.name) key_in_txn = new_bls or new_blspk \ if not add_wrong \ else base58.b58encode(randomString(128).encode()) node_dest = hexToFriendly(node.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, node.name, None, None, None, None, bls_key=key_in_txn, services=None) poolSetExceptOne = list(txnPoolNodeSet) poolSetExceptOne.remove(node) waitNodeDataEquality(looper, node, *poolSetExceptOne) sdk_pool_refresh(looper, sdk_pool_handle) sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias=randomString(5)) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) return new_blspk
def test_state_regenerated_from_ledger(looper, nodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath): """ Node loses its state database but recreates it from ledger after start. Checking ATTRIB txns too since they store some data off ledger too """ trust_anchors = [] for i in range(5): trust_anchors.append(sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'TA' + str(i), TRUST_ANCHOR_STRING)) sdk_add_raw_attribute(looper, sdk_pool_handle, trust_anchors[-1], randomString(6), randomString(10)) for wh in trust_anchors: for i in range(3): sdk_add_new_nym(looper, sdk_pool_handle, wh, 'NP1' + str(i)) ensure_all_nodes_have_same_data(looper, nodeSet) node_to_stop = nodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] assert not node_state.isEmpty state_db_path = node_state._kv.db_path node_to_stop.cleanupOnStopping = False node_to_stop.stop() looper.removeProdable(node_to_stop) ensure_node_disconnected(looper, node_to_stop, nodeSet[:-1]) shutil.rmtree(state_db_path) config_helper = NodeConfigHelper(node_to_stop.name, tconf, chroot=tdir) restarted_node = TestNode( node_to_stop.name, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath, ha=node_to_stop.nodestack.ha, cliha=node_to_stop.clientstack.ha) looper.add(restarted_node) nodeSet[-1] = restarted_node looper.run(checkNodesConnected(nodeSet)) # Need some time as `last_ordered_3PC` is compared too and that is # communicated through catchup waitNodeDataEquality(looper, restarted_node, *nodeSet[:-1]) # Pool is still functional for wh in trust_anchors: sdk_add_new_nym(looper, sdk_pool_handle, wh, 'NP--' + randomString(5)) ensure_all_nodes_have_same_data(looper, nodeSet)
def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint): timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) max_batch_size = chkFreqPatched.Max3PCBatchSize # Send some requests (insufficient for checkpoint), # wait replies and check that current checkpoint is not stable sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2 * max_batch_size) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size, 2) # From the steward send a request creating a user with None role sdk_wallet_user = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 1, 3) # From the created user send a request creating another user. # Dynamic validation of this request must fail since a user with None role cannot create users. # However, the 3PC-batch with the sent request must be ordered. with pytest.raises(RequestRejectedException): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_user) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 2, 4) # Send more requests to cause checkpoint stabilization sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, max_batch_size) # Check that checkpoint is stable now # and verify that requests for it were removed looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 0, 0) # Send more requests to cause new checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint + 1) looper.run(eventually(chkChkpoints, txnPoolNodeSet, 2, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 1, 1)
def testTrustAnchorSuspensionByTrustee( looper, sdk_pool_handle, another_trustee, another_trust_anchor): _, did_ta = another_trust_anchor sdk_suspend_role(looper, sdk_pool_handle, another_trustee, did_ta) with pytest.raises(RequestRejectedException): sdk_add_new_nym(looper, sdk_pool_handle, another_trust_anchor, alias=randomString())
def some_txns_done(tconf, txnPoolNodesLooper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward): for i in range(math.ceil(TOTAL_TXNS / 2)): sdk_add_new_nym(txnPoolNodesLooper, sdk_pool_handle, sdk_wallet_steward, alias='testSteward' + randomString(100)) for i in range(math.floor(TOTAL_TXNS / 2)): sdk_send_random_and_check(txnPoolNodesLooper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5)
def testTrustAnchorSuspendingHimselfByVerkeyFlush(looper, sdk_pool_handle, sdk_wallet_trust_anchor): # The trust anchor has already lost its role due to previous tests, # but it is ok for this test where the trust anchor flushes its verkey # and then he is unable to send NYM due to empty verkey. _, did = sdk_wallet_trust_anchor sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trust_anchor, dest=did, verkey='') with pytest.raises(RequestNackedException) as e: sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trust_anchor) e.match('InsufficientCorrectSignatures')
def sdk_rotate_verkey(looper, sdk_pool_handle, wh, did_of_changer, did_of_changed, seed=None, verkey=None): seed = seed or randomString(32) verkey = looper.loop.run_until_complete( replace_keys_start(wh, did_of_changed, json.dumps({'seed': seed}))) sdk_add_new_nym(looper, sdk_pool_handle, (wh, did_of_changer), dest=did_of_changed, verkey=verkey) looper.loop.run_until_complete( replace_keys_apply(wh, did_of_changed)) return verkey
def testStewardsCanBeAddedOnlyTillAThresholdIsReached(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tconf): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias='testSteward' + randomString(3), role=STEWARD_STRING) with pytest.raises(RequestRejectedException) as e: sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias='testSteward' + randomString(3), role=STEWARD_STRING) error_message = 'New stewards cannot be added by other stewards as there ' \ 'are already {} stewards in the system'.format(tconf.stewardThreshold) assert error_message in e._excinfo[1].args[0]
def testStewardCannotAddNodeWithOutFullFieldsSet(looper, tdir, tconf, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward): """ The case: Steward accidentally sends the NODE txn without full fields set. The expected result: Steward gets NAck response from the pool. """ new_node_name = "Epsilon" new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias='New steward' + randomString( 3), role=STEWARD_STRING) sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \ prepare_new_node_data(tconf, tdir, new_node_name) _, steward_did = new_steward_wallet_handle node_request = looper.loop.run_until_complete( prepare_node_request(steward_did, new_node_name=new_node_name, clientIp=clientIp, clientPort=clientPort, nodeIp=nodeIp, nodePort=nodePort, bls_key=bls_key, sigseed=sigseed, key_proof=key_proof)) # case from the ticket request_json = json.loads(node_request) request_json['operation'][DATA][NODE_PORT + ' '] = \ request_json['operation'][DATA][NODE_PORT] del request_json['operation'][DATA][NODE_PORT] node_request1 = json.dumps(request_json) request_couple = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle, sdk_pool_handle, node_request1) with pytest.raises(RequestNackedException) as e: sdk_get_and_check_replies(looper, [request_couple]) assert 'missed fields - node_port' in e._excinfo[1].args[0] for fn in (NODE_IP, CLIENT_IP, NODE_PORT, CLIENT_PORT): request_json = json.loads(node_request) del request_json['operation'][DATA][fn] node_request2 = json.dumps(request_json) request_couple = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle, sdk_pool_handle, node_request2) # wait NAcks with exact message. it does not works for just 'is missed' # because the 'is missed' will check only first few cases with pytest.raises(RequestNackedException) as e: sdk_get_and_check_replies(looper, [request_couple]) assert 'missed fields' in e._excinfo[1].args[0]
def test_send_same_nyms_only_first_gets_written( looper, do, sdk_pool_handle, sdk_wallet_steward): wh, _ = sdk_wallet_steward seed = randomString(32) did, verkey = looper.loop.run_until_complete( create_and_store_my_did(wh, json.dumps({'seed': seed}))) # request 1 _, did1 = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, dest=did, verkey=verkey) seed = randomString(32) _, verkey = looper.loop.run_until_complete( create_and_store_my_did(wh, json.dumps({'seed': seed}))) # request 2 with pytest.raises(RequestRejectedException) as e: _, did2 = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, dest=did, verkey=verkey) e.match('is neither Trustee nor owner of')
def sdk_node_theta_added(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath, node_config_helper_class, testNodeClass, name=None): new_steward_name = "testClientSteward" + randomString(3) new_node_name = name or "Theta" new_steward_wallet = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, alias=new_steward_name, role=STEWARD_STRING) sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort = \ prepare_new_node_data(tconf, tdir, new_node_name, configClass=node_config_helper_class) # filling node request _, steward_did = new_steward_wallet node_request = looper.loop.run_until_complete( prepare_node_request(steward_did, new_node_name=new_node_name, clientIp=clientIp, clientPort=clientPort, nodeIp=nodeIp, nodePort=nodePort, bls_key=bls_key, sigseed=sigseed, services=[VALIDATOR])) # sending request using 'sdk_' functions request_couple = sdk_sign_and_send_prepared_request(looper, new_steward_wallet, sdk_pool_handle, node_request) # waitng for replies sdk_get_and_check_replies(looper, [request_couple]) new_node = create_and_start_new_node(looper, new_node_name, tdir, sigseed, (nodeIp, nodePort), (clientIp, clientPort), tconf, True, allPluginsPath, testNodeClass, configClass=node_config_helper_class) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) return new_steward_wallet, new_node
def testOnlyAStewardCanAddAnotherSteward(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, sdk_wallet_client): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias='testSteward' + randomString(3), role=STEWARD_STRING) seed = randomString(32) wh, _ = sdk_wallet_client nym_request, steward_did = looper.loop.run_until_complete( prepare_nym_request(sdk_wallet_client, seed, 'testSteward2', 'STEWARD')) request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client, sdk_pool_handle, nym_request) total_timeout = sdk_eval_timeout(1, len(txnPoolNodeSet)) request_couple = sdk_get_replies(looper, [request_couple], total_timeout)[0] with pytest.raises(RequestRejectedException) as e: sdk_check_reply(request_couple) assert 'Only Steward is allowed to do these transactions' in e._excinfo[1].args[0]
def test_add_node_with_existing_data(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_stewards): alias = randomString(5) new_node_name = "Node-" + alias steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_stewards[0], alias="Steward-" + alias, role='STEWARD') # Setting already existing HAs existing_ha = txnPoolNodeSet[0].nodeReg[txnPoolNodeSet[-1].name] existing_cli_ha = txnPoolNodeSet[0].cliNodeReg[txnPoolNodeSet[-1].name + 'C'] # Check for existing alias node_request = create_specific_node_request( looper, steward_wallet_handle, tconf, tdir, txnPoolNodeSet[0].name) request_couple = sdk_sign_and_send_prepared_request(looper, steward_wallet_handle, sdk_pool_handle, node_request) sdk_get_bad_response(looper, [request_couple], RequestRejectedException, "Node's alias must be unique") # Check for existing node HAs node_request = create_specific_node_request( looper, steward_wallet_handle, tconf, tdir, new_node_name, new_node_ip=existing_ha[0], new_node_port=existing_ha[1]) request_couple = sdk_sign_and_send_prepared_request(looper, steward_wallet_handle, sdk_pool_handle, node_request) sdk_get_bad_response(looper, [request_couple], RequestRejectedException, "Node's nodestack addresses must be unique") # Check for existing client HAs node_request = create_specific_node_request( looper, steward_wallet_handle, tconf, tdir, new_node_name, new_client_ip=existing_cli_ha[0], new_client_port=existing_cli_ha[1]) request_couple = sdk_sign_and_send_prepared_request(looper, steward_wallet_handle, sdk_pool_handle, node_request) sdk_get_bad_response(looper, [request_couple], RequestRejectedException, "Node's clientstack addresses must be unique")
def test_add_node_with_not_unique_alias(looper, tdir, tconf, sdk_pool_handle, sdk_wallet_steward, allPluginsPath): new_node_name = "Alpha" new_steward_wallet, steward_did = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias="TEST_STEWARD1", role='STEWARD') with pytest.raises(RequestRejectedException) as e: sdk_add_new_node(looper, sdk_pool_handle, (new_steward_wallet, steward_did), new_node_name, tdir, tconf, allPluginsPath) assert 'existing data has conflicts with request data' in \ e._excinfo[1].args[0] sdk_pool_refresh(looper, sdk_pool_handle)
def testRequestsSize(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, noRetryReq): clients = [] for i in range(4): clients.append(sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward)) numRequests = 250 for (_, nym) in clients: logger.debug("{} sending {} requests".format(nym, numRequests)) sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_steward, numRequests) logger.debug("{} sent {} requests".format(nym, numRequests)) for node in txnPoolNodeSet: logger.debug("{} has requests {} with size {}". format(node, len(node.requests), get_size(node.requests))) for replica in node.replicas.values(): logger.debug("{} has prepares {} with size {}". format(replica, len(replica.prepares), get_size(replica.prepares))) logger.debug("{} has commits {} with size {}". format(replica, len(replica.commits), get_size(replica.commits)))
def sdk_wallet_trust_anchor(looper, sdk_pool_handle, sdk_wallet_trustee): return sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, alias='TA-1', role='TRUST_ANCHOR')
def test_commit_signature_validation_integration(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, sdk_wallet_client, tconf, tdir): ''' All nodes receive PrePrepare1(txn1 for pool_ledger) Nodes 1, 2 ordered txn1 and nodes 3, 4 did not. All nodes receive PrePrepare2(txn2 for domain_ledger) Nodes 3, 4 receive commits from nodes 1, 2 Nodes 3, 4 ordered txn1 Check that all nodes ordered txn2 ''' fast_nodes = txnPoolNodeSet[:2] slow_nodes = txnPoolNodeSet[2:] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1) # create new steward new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias="testClientSteward945", role=STEWARD_STRING) sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \ prepare_new_node_data(tconf, tdir, "new_node") # create node request to add new demote node _, steward_did = new_steward_wallet_handle node_request = looper.loop.run_until_complete( prepare_node_request(steward_did, new_node_name="new_node", clientIp=clientIp, clientPort=clientPort, nodeIp=nodeIp, nodePort=nodePort, bls_key=bls_key, sigseed=sigseed, services=[], key_proof=key_proof)) first_ordered = txnPoolNodeSet[0].master_last_ordered_3PC with ord_delay(slow_nodes): request1 = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle, sdk_pool_handle, node_request) key1 = get_key_from_req(request1[0]) def check_nodes_receive_pp(view_no, seq_no): for node in txnPoolNodeSet: assert node.master_replica._ordering_service.get_preprepare(view_no, seq_no) looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 1)) def check_fast_nodes_ordered_request(): for n in fast_nodes: assert key1 not in n.requests or n.requests[key1].executed for n in slow_nodes: assert not n.requests[key1].executed looper.run(eventually(check_fast_nodes_ordered_request)) request2 = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 2)) def check_nodes_receive_commits(view_no, seq_no): for node in txnPoolNodeSet: assert len(node.master_replica._ordering_service.commits[view_no, seq_no].voters) >= node.f + 1 looper.run(eventually(check_nodes_receive_commits, first_ordered[0], first_ordered[1] + 2)) sdk_get_and_check_replies(looper, [request1]) sdk_get_and_check_replies(looper, [request2])
def sdk_wallet_new_client(looper, sdk_pool_handle, sdk_wallet_steward, sdk_new_client_seed): wh, client_did = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, seed=sdk_new_client_seed) return wh, client_did
def _create_steward(self): return sdk_add_new_nym(self.looper, self.sdk_pool_handle, self.trustee_wallet, role=STEWARD_STRING)
def another_trust_anchor1(looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee): return sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'newTrustAnchor1', TRUST_ANCHOR_STRING)
def prepare(self): self.new_default_wallet = sdk_add_new_nym(self.looper, self.sdk_pool_handle, self.trustee_wallet, role=None) self.default_auth_rule = self.get_default_auth_rule() self.changed_auth_rule = self.get_changed_auth_rule()
def some_transactions_done(looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee)
def _create_trustee(self, trustee_wallet): return sdk_add_new_nym(self.looper, self.sdk_pool_handle, trustee_wallet, role=TRUSTEE_STRING)
def ensurePoolIsOperable(looper, sdk_pool_handle, sdk_wallet_creator): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_creator)
def another_trustee(looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee): return sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'newTrustee', TRUSTEE_STRING)
def testSendAttribSucceedsForExistingDest( looper, sdk_pool_handle, sdk_wallet_trustee): new_wallet = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee) parameters = json.dumps({'name': 'Alice'}) sdk_add_attribute_and_check(looper, sdk_pool_handle, new_wallet, parameters)
def testTrusteeAddingTrustAnchor(looper, sdk_pool_handle, another_trust_anchor): # The new TTrustAnchor adds a NYM sdk_add_new_nym(looper, sdk_pool_handle, another_trust_anchor)
def testSendAttribSucceedsForHexHashWithLettersInBothCases( looper, sdk_pool_handle, sdk_wallet_trustee): new_wallet = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee) parameters = None sdk_add_attribute_and_check(looper, sdk_pool_handle, new_wallet, parameters, xhash='6d4a333838d0ef96756cccC680AF2531075C512502Fb68c5503c63d93de859b3')
def test_different_ledger_request_interleave(tconf, looper, txnPoolNodeSet, sdk_one_node_added, tdir, tdirWithPoolTxns, allPluginsPath, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward): """ Send pool and domain ledger requests such that they interleave, and do view change in between and verify the pool is functional """ new_node = sdk_one_node_added sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) # Send domain ledger requests but don't wait for replies requests = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 2) # Add another node by sending pool ledger request _, new_theta = sdk_node_theta_added(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_steward, allPluginsPath, name='new_theta') # Send more domain ledger requests but don't wait for replies requests.extend(sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 3)) # Do view change without waiting for replies ensure_view_change(looper, nodes=txnPoolNodeSet) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1) # Make sure all requests are completed total_timeout = sdk_eval_timeout(len(requests), len(txnPoolNodeSet)) sdk_get_and_check_replies(looper, requests, timeout=total_timeout) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle) new_steward_wallet, steward_did = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, 'another_ste', role='STEWARD') # Send another pool ledger request (NODE) but don't wait for completion of # request next_node_name = 'next_node' sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \ prepare_new_node_data(tconf, tdir, next_node_name) node_req = looper.loop.run_until_complete( prepare_node_request(steward_did, new_node_name=next_node_name, clientIp=clientIp, clientPort=clientPort, nodeIp=nodeIp, nodePort=nodePort, bls_key=bls_key, sigseed=sigseed, key_proof=key_proof)) sdk_wallet = (new_steward_wallet, steward_did) request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet, sdk_pool_handle, node_req) # Send more domain ledger requests but don't wait for replies request_couples = [request_couple, * sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 5)] # Make sure all requests are completed total_timeout = sdk_eval_timeout(len(request_couples), len(txnPoolNodeSet)) sdk_get_and_check_replies(looper, request_couples, timeout=total_timeout) # Make sure pool is functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_successive_batch_do_no_change_state(looper, tconf, nodeSet, sdk_pool_handle, sdk_wallet_trustee, monkeypatch): """ Send 2 NYM txns in different batches such that the second batch does not change state so that state root remains same, but keep the identifier and reqId different. Make sure the first request is not ordered by the primary before PRE-PREPARE for the second is sent. Also check reject and commit :return: """ # Disable view change during this test for n in nodeSet: n.nodeIbStasher.delay(icDelay()) # Delay only first PRE-PREPARE delay_cm_duration = 10 def delay_commits(wrappedMsg): msg, sender = wrappedMsg if isinstance(msg, Commit) and msg.instId == 0: return delay_cm_duration def check_verkey(i, vk): for node in nodeSet: data = node.idrCache.getNym(i, isCommitted=True) assert data[VERKEY] == vk def check_uncommitted(count): for node in nodeSet: assert len(node.idrCache.un_committed) == count for node in nodeSet: for rpl in node.replicas: monkeypatch.setattr(rpl, '_request_missing_three_phase_messages', lambda *x, **y: None) wh, did = sdk_wallet_trustee seed = randomString(32) (new_did, verkey) = looper.loop.run_until_complete( create_and_store_my_did(wh, json.dumps({'seed': seed}))) sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, dest=new_did, verkey=verkey) for node in nodeSet: node.nodeIbStasher.delay(delay_commits) # Setting the same verkey thrice but in different batches with different # request ids for _ in range(3): verkey = sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_did, new_did, seed) logger.debug('{} rotates his key to {}'.format(new_did, verkey)) # Number of uncommitted entries is 0 looper.run(eventually(check_uncommitted, 0)) check_verkey(new_did, verkey) # Setting the verkey to `x`, then `y` and then back to `x` but in different # batches with different request ids. The idea is to change # state root to `t` then `t'` and then back to `t` and observe that no # errors are encountered seed = randomString(32) (new_client_did, verkey) = looper.loop.run_until_complete( create_and_store_my_did(wh, json.dumps({'seed': seed}))) sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, dest=new_client_did, verkey=verkey) x_seed = randomString(32) verkey = sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_client_did, new_client_did, x_seed) logger.debug('{} rotates his key to {}'.format(new_client_did, verkey)) y_seed = randomString(32) sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_client_did, new_client_did, y_seed) logger.debug('{} rotates his key to {}'.format(new_client_did, verkey)) sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_client_did, new_client_did, x_seed) logger.debug('{} rotates his key to {}'.format(new_client_did, verkey)) # Number of uncommitted entries is 0 looper.run(eventually(check_uncommitted, 0)) check_verkey(new_client_did, verkey) monkeypatch.undo() # Delay COMMITs so that IdrCache can be checked for correct # number of entries uncommitteds = {} methods = {} for node in nodeSet: cache = node.idrCache # type: IdrCache uncommitteds[cache._name] = [] # Since the post batch creation handler is registered (added to a list), # find it and patch it dh = node.get_req_handler(DOMAIN_LEDGER_ID) for i, handler in enumerate(dh.post_batch_creation_handlers): # Find the cache's post create handler, not hardcoding names of # class or functions as they can change with refactoring. if handler.__func__.__qualname__ == '{}.{}'.format( cache.__class__.__name__, cache.currentBatchCreated.__name__): cre = dh.post_batch_creation_handlers[i] break com = cache.onBatchCommitted methods[cache._name] = (cre, com) # Patch methods to record and check roots after commit def patched_cre(self, stateRoot): uncommitteds[self._name].append(stateRoot) return methods[self._name][0](stateRoot) def patched_com(self, stateRoot): assert uncommitteds[self._name][0] == stateRoot rv = methods[self._name][1](stateRoot) uncommitteds[self._name] = uncommitteds[self._name][1:] return rv dh.post_batch_creation_handlers[i] = types.MethodType( patched_cre, cache) cache.onBatchCommitted = types.MethodType(patched_com, cache) # Set verkey of multiple identities more = 5 keys = {} reqs = [] for _ in range(more): seed = randomString(32) nym_request, new_did = looper.loop.run_until_complete( prepare_nym_request(sdk_wallet_trustee, seed, None, None)) keys[new_did] = json.loads(nym_request)['operation']['verkey'] reqs.append( sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee, sdk_pool_handle, nym_request)) looper.runFor(.01) # Correct number of uncommitted entries looper.run(eventually(check_uncommitted, more, retryWait=1)) sdk_get_and_check_replies(looper, reqs) # Number of uncommitted entries is 0 looper.run(eventually(check_uncommitted, 0)) # The verkeys are correct for i, v in keys.items(): check_verkey(i, v) waitNodeDataEquality(looper, nodeSet[0], *nodeSet[1:]) for _ in range(3): seed = randomString(32) nym_request, new_did = looper.loop.run_until_complete( prepare_nym_request(sdk_wallet_trustee, seed, None, None)) reqs.append( sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee, sdk_pool_handle, nym_request)) looper.runFor(.01) # Correct number of uncommitted entries looper.run(eventually(check_uncommitted, 3, retryWait=1)) # Check batch reject for node in nodeSet: cache = node.idrCache initial = cache.un_committed cache.batchRejected() # After reject, last entry is removed assert cache.un_committed == initial[:-1] root = cache.un_committed[0][0] cache.onBatchCommitted(root) # Calling commit with same root results in Assertion error with pytest.raises(AssertionError): cache.onBatchCommitted(root)
def testStewardCreatesAEndorser(looper, sdk_pool_handle, sdk_wallet_steward): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, role=ENDORSER_STRING)
def test_commit_signature_validation_integration(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, sdk_wallet_client, tconf, tdir): ''' All nodes receive PrePrepare1(txn1 for pool_ledger) Nodes 1, 2 ordered txn1 and nodes 3, 4 did not. All nodes receive PrePrepare2(txn2 for domain_ledger) Nodes 3, 4 receive commits from nodes 1, 2 Nodes 3, 4 ordered txn1 Check that all nodes ordered txn2 ''' fast_nodes = txnPoolNodeSet[:2] slow_nodes = txnPoolNodeSet[2:] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1) # create new steward new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias="testClientSteward945", role=STEWARD_STRING) sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \ prepare_new_node_data(tconf, tdir, "new_node") # create node request to add new demote node _, steward_did = new_steward_wallet_handle node_request = looper.loop.run_until_complete( prepare_node_request(steward_did, new_node_name="new_node", clientIp=clientIp, clientPort=clientPort, nodeIp=nodeIp, nodePort=nodePort, bls_key=bls_key, sigseed=sigseed, services=[], key_proof=key_proof)) first_ordered = txnPoolNodeSet[0].master_last_ordered_3PC with ord_delay(slow_nodes): request1 = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle, sdk_pool_handle, node_request) key1 = get_key_from_req(request1[0]) def check_nodes_receive_pp(view_no, seq_no): for node in txnPoolNodeSet: assert node.master_replica.getPrePrepare(view_no, seq_no) looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 1)) def check_fast_nodes_ordered_request(): for n in fast_nodes: assert key1 not in n.requests or n.requests[key1].executed for n in slow_nodes: assert not n.requests[key1].executed looper.run(eventually(check_fast_nodes_ordered_request)) request2 = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 2)) def check_nodes_receive_commits(view_no, seq_no): for node in txnPoolNodeSet: assert len(node.master_replica.commits[view_no, seq_no].voters) >= node.f + 1 looper.run(eventually(check_nodes_receive_commits, first_ordered[0], first_ordered[1] + 2)) sdk_get_and_check_replies(looper, [request1]) sdk_get_and_check_replies(looper, [request2])
def another_steward1(looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee): return sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'newSteward1', STEWARD_STRING)
def sdk_wallet_endorser(looper, sdk_pool_handle, sdk_wallet_trustee): return sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, alias='TA-1', role='ENDORSER')
def new_client_wallet(self): return sdk_add_new_nym(self.looper, self.sdk_pool_handle, self.trustee_wallet, role=IDENTITY_OWNER)
def testTrusteeAddingSteward(looper, sdk_pool_handle, another_steward): # The new Steward adds a TRUST_ANCHOR sdk_add_new_nym(looper, sdk_pool_handle, another_steward, role=TRUST_ANCHOR_STRING)
def test_audit_ledger_view_change(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward, initial_domain_size, initial_pool_size, initial_config_size, tdir, tconf, allPluginsPath, view_no, pp_seq_no, initial_seq_no, monkeypatch): ''' 1. Send a NODE transaction and add a 7th Node for adding a new instance, but delay Ordered messages. 2. Send a NYM txn. 3. Reset delays in executing force_process_ordered 4. Check that an audit txn for the NYM txn uses primary list from uncommitted audit with a new list of primaries. ''' expected_pp_seq_no = 0 other_nodes = txnPoolNodeSet[:-1] slow_node = txnPoolNodeSet[-1] # Add a new steward for creating a new node new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias="newSteward", role=STEWARD_STRING) audit_size_initial = [node.auditLedger.size for node in txnPoolNodeSet] ordereds = [] monkeypatch.setattr(slow_node, 'try_processing_ordered', lambda msg: ordereds.append(msg)) with delay_rules([n.nodeIbStasher for n in txnPoolNodeSet], icDelay()): # Send NODE txn fo 7th node new_node = sdk_add_new_node(looper, sdk_pool_handle, new_steward_wallet_handle, "Theta", tdir, tconf, allPluginsPath) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(other_nodes + [new_node])) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) check_audit_ledger_updated(audit_size_initial, [slow_node], audit_txns_added=0) looper.run( eventually(check_audit_ledger_uncommitted_updated, audit_size_initial, [slow_node], 2)) def patch_force_process_ordered(): for msg in list(ordereds): slow_node.replicas[msg.instId].outBox.append(msg) ordereds.remove(msg) monkeypatch.undo() slow_node.force_process_ordered() assert ordereds monkeypatch.setattr(slow_node, 'force_process_ordered', patch_force_process_ordered) looper.run( eventually( lambda: assertExp(all(n.viewNo == 1 for n in txnPoolNodeSet)))) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) looper.run(eventually(lambda: assertExp(not ordereds))) lpps = set([n.master_replica.lastPrePrepareSeqNo for n in txnPoolNodeSet]) assert len(lpps) == 1 expected_pp_seq_no = lpps.pop() for node in txnPoolNodeSet: last_txn = node.auditLedger.get_last_txn() last_txn['txn']['data'][ 'primaries'] = node._get_last_audited_primaries() check_audit_txn(txn=last_txn, view_no=view_no + 1, pp_seq_no=expected_pp_seq_no, seq_no=initial_seq_no + 4, txn_time=node.master_replica._ordering_service. last_accepted_pre_prepare_time, txn_roots={ DOMAIN_LEDGER_ID: node.getLedger(DOMAIN_LEDGER_ID).tree.root_hash }, state_roots={ DOMAIN_LEDGER_ID: node.getState(DOMAIN_LEDGER_ID).committedHeadHash }, pool_size=initial_pool_size + 1, domain_size=initial_domain_size + 2, config_size=initial_config_size, last_pool_seqno=2, last_domain_seqno=1, last_config_seqno=None, primaries=node.write_manager.future_primary_handler. get_last_primaries() or node.primaries)
def testSendGetNymSucceedsForExistingUuidDest(looper, sdk_pool_handle, sdk_wallet_trustee): new_wallet = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee) get_nym(looper, sdk_pool_handle, sdk_wallet_trustee, new_wallet[1])
def testStewardCreatesAnotherTrustAnchor(looper, sdk_pool_handle, sdk_wallet_steward): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, role=TRUST_ANCHOR_STRING)
def send_txns_invalid(): with pytest.raises(RequestRejectedException, match='Rule for this action is'): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_client)
def sdk_user_wallet_a(nodeSet, sdk_wallet_endorser, sdk_pool_handle, looper): return sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_endorser, alias='userA', skipverkey=True)
def test_non_steward_cannot_create_trust_anchor( nodeSet, looper, sdk_pool_handle, sdk_wallet_steward): sdk_wallet_client = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward) with pytest.raises(RequestRejectedException) as e: sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_client, role=TRUST_ANCHOR_STRING) e.match('None role cannot')
def sdk_user_wallet_a(nodeSet, sdk_wallet_trust_anchor, sdk_pool_handle, looper, trustAnchor): return sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trust_anchor, alias='userA', skipverkey=True)
def test_different_ledger_request_interleave(tconf, looper, txnPoolNodeSet, sdk_one_node_added, tdir, tdirWithPoolTxns, allPluginsPath, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward): """ Send pool and domain ledger requests such that they interleave, and do view change in between and verify the pool is functional """ new_node = sdk_one_node_added sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) # Send domain ledger requests but don't wait for replies requests = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 2) # Add another node by sending pool ledger request _, new_theta = sdk_node_theta_added(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_steward, allPluginsPath, name='new_theta') # Send more domain ledger requests but don't wait for replies requests.extend( sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 3)) # Do view change without waiting for replies ensure_view_change(looper, nodes=txnPoolNodeSet) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1) # Make sure all requests are completed total_timeout = sdk_eval_timeout(len(requests), len(txnPoolNodeSet)) sdk_get_and_check_replies(looper, requests, timeout=total_timeout) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle) new_steward_wallet, steward_did = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, 'another_ste', role='STEWARD') # Send another pool ledger request (NODE) but don't wait for completion of # request next_node_name = 'next_node' sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort = \ prepare_new_node_data(tconf, tdir, next_node_name) node_req = looper.loop.run_until_complete( prepare_node_request(steward_did, next_node_name, clientIp, clientPort, nodeIp, nodePort, bls_key, sigseed)) sdk_wallet = (new_steward_wallet, steward_did) request_couple = sdk_sign_and_send_prepared_request( looper, sdk_wallet, sdk_pool_handle, node_req) # Send more domain ledger requests but don't wait for replies request_couples = [ request_couple, *sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 5) ] # Make sure all requests are completed total_timeout = sdk_eval_timeout(len(request_couples), len(txnPoolNodeSet)) sdk_get_and_check_replies(looper, request_couples, timeout=total_timeout) # Make sure pool is functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def check_view_change_adding_new_node(looper, tdir, tconf, allPluginsPath, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward, slow_nodes=[], delay_commit=False, delay_pre_prepare=False, trigger_view_change_manually=False): # Pre-requisites: viewNo=3, Primary is Node4 for viewNo in range(1, 4): trigger_view_change(txnPoolNodeSet) waitForViewChange(looper, txnPoolNodeSet, viewNo) ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30) # Delay 3PC messages on slow nodes fast_nodes = [node for node in txnPoolNodeSet if node not in slow_nodes] all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet] slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes] delayers = [] if delay_pre_prepare: delayers.append(ppDelay()) delayers.append(msg_rep_delay(types_to_delay=[PREPREPARE])) if delay_commit: delayers.append(cDelay()) # add a new Steward before delaying. Otherwise the slow node may reject NODE client reqs # as it can not authenticate it due to lack of Steward txn applied new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias='New_Steward', role=STEWARD_STRING) # delay NewView message to make sure that all old nodes started view change, # but finish the view change when no Commits are delayed (otherwise slow node will not be able to select backup primaries) with delay_rules(all_stashers, nv_delay()): with delay_rules_without_processing(slow_stashers, *delayers): # Add Node5 new_node = sdk_add_new_node( looper, sdk_pool_handle, new_steward_wallet_handle, 'Epsilon', tdir, tconf, allPluginsPath, autoStart=True, nodeClass=TestNode, do_post_node_creation=None, services=[VALIDATOR], wait_till_added=True) looper.run(checkNodesConnected(fast_nodes + [new_node])) old_set = list(txnPoolNodeSet) txnPoolNodeSet.append(new_node) if trigger_view_change_manually: trigger_view_change(txnPoolNodeSet) # make sure view change is started and finished eventually waitForViewChange(looper, old_set, 4) ensureElectionsDone(looper, old_set) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def sdk_wallet_new_steward(looper, sdk_pool_handle, sdk_wallet_steward): wh, client_did = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias='new_steward_qwerty', role='STEWARD') return wh, client_did
def test_non_trust_anchor_cannot_create_user( nodeSet, looper, sdk_pool_handle, sdk_wallet_steward): sdk_wallet_client = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward) with pytest.raises(RequestRejectedException) as e: sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_client) e.match('None role cannot')
def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint): timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) max_batch_size = chkFreqPatched.Max3PCBatchSize # Send some requests (insufficient for checkpoint), # wait replies and check that current checkpoint is not stable sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2 * max_batch_size) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size, 2) # From the steward send a request creating a user with None role sdk_wallet_user = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 1, 3) # From the created user send a request creating another user. # Dynamic validation of this request must fail since a user with None role cannot create users. # However, the 3PC-batch with the sent request must be ordered. with pytest.raises(RequestRejectedException): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_user) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 0, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 2, 4) # Send more requests to cause checkpoint stabilization sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, max_batch_size) # Check that checkpoint is stable now # and verify that requests for it were removed looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 5, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 0, 0) # Send more requests to cause new checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint + 1) looper.run( eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint, 10, retryWait=1, timeout=timeout)) checkRequestCounts(txnPoolNodeSet, 1, 1)