def run(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test(network, args, batch_size=1) network = test(network, args, batch_size=10) network = test(network, args, batch_size=100) network = test(network, args, batch_size=1000) network = test(network, args, batch_size=1000, write_key_divisor=10) network = test(network, args, batch_size=1000, write_size_multiplier=10) network = test( network, args, batch_size=1000, write_key_divisor=10, write_size_multiplier=10, )
def gov(args): for node in args.nodes: node.rpc_interfaces.update(infra.interfaces.make_secondary_interface()) with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network.consortium.set_authenticate_session(args.authenticate_session) test_create_endpoint(network, args) test_consensus_status(network, args) test_member_data(network, args) network = test_all_members(network, args) test_quote(network, args) test_user(network, args) test_jinja_templates(network, args) test_no_quote(network, args) test_node_data(network, args) test_ack_state_digest_update(network, args) test_invalid_client_signature(network, args) test_each_node_cert_renewal(network, args) test_all_nodes_cert_renewal(network, args) test_service_cert_renewal(network, args) test_service_cert_renewal_extended(network, args)
def run_corrupted_ledger(args): txs = app.LoggingTxs("user0") with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs, ) as network: network.start_and_open(args) network = test_recover_service_truncated_ledger( network, args, corrupt_first_tx=True ) network = test_recover_service_truncated_ledger( network, args, corrupt_last_tx=True ) network = test_recover_service_truncated_ledger( network, args, corrupt_first_sig=True ) network.stop_all_nodes() # Make sure ledger can be read once recovered (i.e. ledger corruption does not affect recovered ledger) for node in network.nodes: ledger = ccf.ledger.Ledger(node.remote.ledger_paths(), committed_only=False) _, last_seqno = ledger.get_latest_public_state() LOG.info( f"Successfully read ledger for node {node.local_node_id} up to seqno {last_seqno}" )
def run_nobuiltins(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) nobuiltins.test_nobuiltins_endpoints(network, args)
def run(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_custom_auth(network, args)
def run_request_object(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_request_object_api(network, args)
def run_join_old_snapshot(args): txs = app.LoggingTxs("user0") nodes = ["local://localhost"] with tempfile.TemporaryDirectory() as tmp_dir: with infra.network.network( nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs, ) as network: network.start_and_open(args) primary, _ = network.find_primary() # First, retrieve and save one committed snapshot txs.issue(network, number_txs=args.snapshot_tx_interval) old_committed_snapshots = network.get_committed_snapshots(primary) copy( os.path.join( old_committed_snapshots, os.listdir(old_committed_snapshots)[0] ), tmp_dir, ) # Then generate another newer snapshot, and add two more nodes from it txs.issue(network, number_txs=args.snapshot_tx_interval) for _ in range(0, 2): new_node = network.create_node("local://localhost") network.join_node( new_node, args.package, args, from_snapshot=True, ) network.trust_node(new_node, args) # Kill primary and wait for a new one: new primary is # guaranteed to have started from the new snapshot primary.stop() network.wait_for_new_primary(primary) # Start new node from the old snapshot try: new_node = network.create_node("local://localhost") network.join_node( new_node, args.package, args, from_snapshot=True, snapshots_dir=tmp_dir, timeout=3, ) except infra.network.StartupSnapshotIsOld: pass
def run(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_historical_query_range(network, args)
def run(args): with infra.service_load.load() as load: with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, service_load=load, ) as network: check = infra.checker.Checker() network.start_and_open(args) current_view = None primary, current_view = network.find_primary() # Number of nodes F to stop until network cannot make progress nodes_to_stop = math.ceil(len(args.nodes) / 2) if args.consensus == "BFT": nodes_to_stop = math.ceil(len(args.nodes) / 3) primary_is_known = True for node_to_stop in range(nodes_to_stop): primary, current_view = network.find_primary() LOG.debug( "Commit new transactions, primary:{}, current_view:{}". format(primary.local_node_id, current_view)) with primary.client("user0") as c: res = c.post( "/app/log/private", { "id": current_view, "msg": "This log is committed in view {}".format( current_view), }, ) check(res, result=True) LOG.debug( "Waiting for transaction to be committed by all nodes") network.wait_for_all_nodes_to_commit( tx_id=TxID(res.view, res.seqno)) try: test_kill_primary_no_reqs(network, args) except PrimaryNotFound: if node_to_stop < nodes_to_stop - 1: raise else: primary_is_known = False assert not primary_is_known, "Primary is still known" LOG.success("Test ended successfully.")
def run(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_host_process_launch(network, args) network = test_host_process_launch_many(network, args)
def run_tls_san_checks(args): with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, ) as network: args.common_read_only_ledger_dir = None # Reset from previous test network.start_and_open(args) network.verify_service_certificate_validity_period( args.initial_service_cert_validity_days) LOG.info("Check SAN value in TLS certificate") dummy_san = "*.dummy.com" new_node = network.create_node( infra.interfaces.HostSpec( rpc_interfaces={ infra.interfaces.PRIMARY_RPC_INTERFACE: infra.interfaces. RPCInterface(endorsement=infra.interfaces.Endorsement( authority=infra.interfaces.EndorsementAuthority.Node)) })) args.subject_alt_names = [f"dNSName:{dummy_san}"] network.join_node(new_node, args.package, args) sans = infra.crypto.get_san_from_pem_cert( new_node.get_tls_certificate_pem()) assert len(sans) == 1, "Expected exactly one SAN" assert sans[0].value == dummy_san LOG.info( "A node started with no specified SAN defaults to public RPC host") dummy_public_rpc_host = "123.123.123.123" args.subject_alt_names = [] new_node = network.create_node( infra.interfaces.HostSpec( rpc_interfaces={ infra.interfaces.PRIMARY_RPC_INTERFACE: infra.interfaces.RPCInterface( public_host=dummy_public_rpc_host, endorsement=infra.interfaces.Endorsement( authority=infra.interfaces.EndorsementAuthority. Node), ) })) network.join_node(new_node, args.package, args) # Cannot trust the node here as client cannot authenticate dummy public IP in cert with open( os.path.join(network.common_dir, f"{new_node.local_node_id}.pem"), encoding="utf-8", ) as self_signed_cert: sans = infra.crypto.get_san_from_pem_cert(self_signed_cert.read()) assert len(sans) == 1, "Expected exactly one SAN" assert sans[0].value == ipaddress.ip_address(dummy_public_rpc_host)
def run_authn(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_jwt_auth(network, args) network = test_multi_auth(network, args) network = test_role_based_access(network, args)
def run(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_missing_signature_header(network, args) network = test_corrupted_signature(network, args) network = test_governance(network, args)
def run_content_types(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_content_types(network, args) network = test_accept_header(network, args) network = test_supported_methods(network, args) network = test_unknown_path(network, args)
def run(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_primary(network, args) network = test_network_node_info(network, args) network = test_node_ids(network, args) network = test_memory(network, args)
def run_manual(args): with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb ) as network: network.start_and_open(args) test_jwt_key_initial_refresh(network, args) # Check that initial refresh also works on backups primary, _ = network.find_primary() primary.stop() network.wait_for_new_primary(primary) test_jwt_key_initial_refresh(network, args)
def js_gov(args): with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb ) as network: network.start_and_open(args) governance_js.test_proposal_validation(network, args) governance_js.test_proposal_storage(network, args) governance_js.test_proposal_withdrawal(network, args) governance_js.test_ballot_storage(network, args) governance_js.test_pure_proposals(network, args) governance_js.test_proposals_with_votes(network, args) governance_js.test_vote_failure_reporting(network, args) governance_js.test_operator_proposals_and_votes(network, args) governance_js.test_apply(network, args) governance_js.test_set_constitution(network, args)
def run_limits(args): if "v8" in args.package: LOG.warning( f"Skipping run_limits for {args.package} as heap and stack limits are not yet enforced" ) # See https://github.com/microsoft/CCF/issues/3324 return with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_stack_size_limit(network, args) network = test_heap_size_limit(network, args)
def run(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) test_verify_quotes(network, args) test_add_node_with_bad_code(network, args) # NB: Assumes the current nodes are still using args.package, so must run before test_proposal_invalidation test_proposal_invalidation(network, args) test_update_all_nodes(network, args) # Run again at the end to confirm current nodes are acceptable test_verify_quotes(network, args)
def run(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) network = test_module_import(network, args) network = test_dynamic_module_import(network, args) network = test_bytecode_cache(network, args) network = test_app_bundle(network, args) network = test_dynamic_endpoints(network, args) if "v8" not in args.package: # endpoint calls fail with "Cannot access \'logMap\' before init..." # as if the const logMap wasn't preserved/captured network = test_npm_app(network, args)
def run_to_destruction(args): with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) LOG.warning("About to issue transactions until destruction") try: wsm = 5000 while True: LOG.info(f"Trying with writes scaled by {wsm}") network = test(network, args, batch_size=10, write_size_multiplier=wsm) if wsm > 1000000: LOG.error( f"Run to destruction still hasn't caused exception with write sizes multiplied by {wsm}. Infinite loop, or not actually submitting?" ) raise ValueError(wsm) else: wsm += 50000 # Grow very quickly, expect to fail on the second iteration except Exception as e: timeout = 10 LOG.info("Large write set caused an exception, as expected") LOG.info(f"Exception was: {e}") LOG.info(f"Polling for {timeout}s for node to terminate") end_time = time.time() + timeout while time.time() < end_time: time.sleep(0.1) exit_code = network.nodes[0].remote.remote.proc.poll() if exit_code is not None: LOG.info(f"Node terminated with exit code {exit_code}") assert exit_code != 0 break if time.time() > end_time: raise TimeoutError( f"Node took longer than {timeout}s to terminate") from e network.ignore_errors_on_shutdown()
def run_auto(args): with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb ) as network: network.start_and_open(args) test_jwt_endpoint(network, args) test_jwt_without_key_policy(network, args) if args.enclave_type != "virtual": test_jwt_with_sgx_key_policy(network, args) test_jwt_with_sgx_key_filter(network, args) test_jwt_key_auto_refresh(network, args) # Check that auto refresh also works on backups primary, _ = network.find_primary() primary.stop() network.wait_for_new_primary(primary) test_jwt_key_auto_refresh(network, args)
def run(args): with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb ) as network: for node in network.nodes: node.curl = True network.start_and_open(args) primary, _ = network.find_primary() uncommitted_ledger_dir, committed_ledger_dirs = list(primary.get_ledger()) cmd = [ "python", args.ledger_tutorial, *committed_ledger_dirs, uncommitted_ledger_dir, ] rc = infra.proc.ccall(*cmd).returncode assert rc == 0, f"Failed to run tutorial script: {rc}"
def run_migration_tests(args): if args.reconfiguration_type != "OneTransaction": return with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, ) as network: network.start_and_open(args) test_migration_2tx_reconfiguration(network, args) primary, _ = network.find_primary() new_node = network.nodes[-1] ledger_paths = primary.remote.ledger_paths() learner_id = new_node.node_id check_2tx_ledger(ledger_paths, learner_id)
def run(args): txs = app.LoggingTxs("user0") with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs, init_partitioner=True, ) as network: network.start_and_open(args) test_invalid_partitions(network, args) test_partition_majority(network, args) test_isolate_primary_from_one_backup(network, args) test_new_joiner_helps_liveness(network, args) for n in range(5): test_isolate_and_reconnect_primary(network, args, iteration=n)
def run_2tx_reconfig_tests(args): if not args.include_2tx_reconfig: return local_args = args if args.reconfiguration_type != "TwoTransaction": local_args.reconfiguration_type = "TwoTransaction" with infra.network.network( local_args.nodes, local_args.binary_dir, local_args.debug_nodes, local_args.perf_nodes, pdb=local_args.pdb, init_partitioner=True, ) as network: network.start_and_open(local_args) test_learner_does_not_take_part(network, local_args)
def run(args): txs = app.LoggingTxs("user0") with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs, ) as network: network.start_and_open(args) test_version(network, args) if args.consensus != "BFT": test_join_straddling_primary_replacement(network, args) test_node_replacement(network, args) test_add_node_from_backup(network, args) test_add_node(network, args) test_add_node_on_other_curve(network, args) test_retire_backup(network, args) test_add_as_many_pending_nodes(network, args) test_add_node(network, args) test_retire_primary(network, args) test_add_node_with_read_only_ledger(network, args) test_add_node_from_snapshot(network, args) test_add_node_from_snapshot(network, args, from_backup=True) test_add_node_from_snapshot(network, args, copy_ledger_read_only=False) test_node_filter(network, args) test_retiring_nodes_emit_at_most_one_signature(network, args) if args.reconfiguration_type == "TwoTransaction": test_learner_catches_up(network, args) test_service_config_endpoint(network, args) test_node_certificates_validity_period(network, args) test_add_node_invalid_validity_period(network, args)
def run_file_operations(args): with tempfile.TemporaryDirectory() as tmp_dir: txs = app.LoggingTxs("user0") with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs, ) as network: args.common_read_only_ledger_dir = tmp_dir network.start_and_open(args) test_save_committed_ledger_files(network, args) test_parse_snapshot_file(network, args) test_forced_ledger_chunk(network, args) test_forced_snapshot(network, args) primary, _ = network.find_primary() network.stop_all_nodes() test_split_ledger_on_stopped_network(primary, args)
def service_startups(args): LOG.info("Starting service with insufficient number of recovery members") args.initial_member_count = 2 args.initial_recovery_member_count = 0 args.initial_operator_count = 1 args.ledger_recovery_timeout = 5 with infra.network.network(args.nodes, args.binary_dir, pdb=args.pdb) as network: try: network.start_and_open(args) assert False, "Service cannot be opened with no recovery members" except infra.proposal.ProposalNotAccepted: primary, _ = network.find_primary() network.consortium.check_for_service( primary, infra.network.ServiceStatus.OPENING) LOG.success( "Service could not be opened with insufficient number of recovery members" ) LOG.info( "Starting service with a recovery operator member, a non-recovery operator member and a non-recovery non-operator member" ) args.initial_member_count = 3 args.initial_recovery_member_count = 1 args.initial_operator_count = 2 with infra.network.network(args.nodes, args.binary_dir, pdb=args.pdb) as network: network.start_and_open(args) LOG.info( "Starting service with a recovery operator member, a recovery non-operator member and a non-recovery non-operator member" ) args.initial_member_count = 3 args.initial_recovery_member_count = 2 args.initial_operator_count = 1 with infra.network.network(args.nodes, args.binary_dir, pdb=args.pdb) as network: network.start_and_open(args)
def run(args): # This is deliberately 5, because the rest of the test depends on this # to grow a prefix and allow just enough nodes to resume to reach the # desired election result. Conversion to a general f isn't trivial. hosts = ["local://localhost"] * 5 with infra.network.network(hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_open(args) primary, backups = network.find_nodes() # Suspend three of the backups to prevent commit backups[1].suspend() backups[2].suspend() backups[3].stop() committable_txs = [] # Run some transactions that can't be committed now with primary.client("user0") as uc: for i in range(3): committable_txs.append( uc.post("/app/log/private", { "id": 100 + i, "msg": "Hello world" })) last_tx = committable_txs[-1] sig_view, sig_seqno = last_tx.view, last_tx.seqno + 1 with backups[0].client() as bc: wait_for_pending(bc, sig_view, sig_seqno) # Suspend the final backup and run some transactions which only the partitioned # primary hears, which should be discarded by the new primary # NB: We can't guarantee that these will be discarded. Since we can't control # what order the queued actions occur in after resuming, they may be appended # before an election is called. They key assertion is that this primary is able # to rejoin the network whatever happens, even when (in the usual case) they # hold a suffix which has been discarded. backups[0].suspend() post_partition_txs = [] with primary.client("user0") as uc: for i in range(3): post_partition_txs.append( uc.post("/app/log/private", { "id": 100 + i, "msg": "Hello world" })) # Sleep long enough that this primary should be instantly replaced when nodes wake sleep_time = 2 * args.election_timeout_ms / 1000 LOG.info(f"Sleeping {sleep_time}s") time.sleep(sleep_time) # Suspend the primary, resume other backups primary.suspend() backups[0].resume() backups[1].resume() backups[2].resume() new_primary, _ = network.wait_for_new_primary(primary, timeout_multiplier=10) with new_primary.client("user0") as uc: # Check that uncommitted but committable suffix is preserved check_commit = infra.checker.Checker(uc) for tx in committable_txs: check_commit(tx) # Check that new transactions can be committed with new_primary.client("user0") as uc: for i in range(3): r = uc.post("/app/log/private", { "id": 100 + i, "msg": "Hello world" }) assert r.status_code == 200 uc.wait_for_commit(r) # Resume original primary, check that they rejoin correctly, including new transactions primary.resume() network.wait_for_node_commit_sync(timeout=16)