def run(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test(network, args, batch_size=1)
network = test(network, args, batch_size=10)
network = test(network, args, batch_size=100)
network = test(network, args, batch_size=1000)
network = test(network, args, batch_size=1000, write_key_divisor=10)
network = test(network,
args,
batch_size=1000,
write_size_multiplier=10)
network = test(
network,
args,
batch_size=1000,
write_key_divisor=10,
write_size_multiplier=10,
)
def gov(args):
for node in args.nodes:
node.rpc_interfaces.update(infra.interfaces.make_secondary_interface())
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network.consortium.set_authenticate_session(args.authenticate_session)
test_create_endpoint(network, args)
test_consensus_status(network, args)
test_member_data(network, args)
network = test_all_members(network, args)
test_quote(network, args)
test_user(network, args)
test_jinja_templates(network, args)
test_no_quote(network, args)
test_node_data(network, args)
test_ack_state_digest_update(network, args)
test_invalid_client_signature(network, args)
test_each_node_cert_renewal(network, args)
test_all_nodes_cert_renewal(network, args)
test_service_cert_renewal(network, args)
test_service_cert_renewal_extended(network, args)
def run_corrupted_ledger(args):
txs = app.LoggingTxs("user0")
with infra.network.network(
args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb,
txs=txs,
) as network:
network.start_and_open(args)
network = test_recover_service_truncated_ledger(
network, args, corrupt_first_tx=True
)
network = test_recover_service_truncated_ledger(
network, args, corrupt_last_tx=True
)
network = test_recover_service_truncated_ledger(
network, args, corrupt_first_sig=True
)
network.stop_all_nodes()
# Make sure ledger can be read once recovered (i.e. ledger corruption does not affect recovered ledger)
for node in network.nodes:
ledger = ccf.ledger.Ledger(node.remote.ledger_paths(), committed_only=False)
_, last_seqno = ledger.get_latest_public_state()
LOG.info(
f"Successfully read ledger for node {node.local_node_id} up to seqno {last_seqno}"
)
def run_nobuiltins(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
nobuiltins.test_nobuiltins_endpoints(network, args)
def run(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_custom_auth(network, args)
def run_request_object(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_request_object_api(network, args)
def run_join_old_snapshot(args):
txs = app.LoggingTxs("user0")
nodes = ["local://localhost"]
with tempfile.TemporaryDirectory() as tmp_dir:
with infra.network.network(
nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb,
txs=txs,
) as network:
network.start_and_open(args)
primary, _ = network.find_primary()
# First, retrieve and save one committed snapshot
txs.issue(network, number_txs=args.snapshot_tx_interval)
old_committed_snapshots = network.get_committed_snapshots(primary)
copy(
os.path.join(
old_committed_snapshots, os.listdir(old_committed_snapshots)[0]
),
tmp_dir,
)
# Then generate another newer snapshot, and add two more nodes from it
txs.issue(network, number_txs=args.snapshot_tx_interval)
for _ in range(0, 2):
new_node = network.create_node("local://localhost")
network.join_node(
new_node,
args.package,
args,
from_snapshot=True,
)
network.trust_node(new_node, args)
# Kill primary and wait for a new one: new primary is
# guaranteed to have started from the new snapshot
primary.stop()
network.wait_for_new_primary(primary)
# Start new node from the old snapshot
try:
new_node = network.create_node("local://localhost")
network.join_node(
new_node,
args.package,
args,
from_snapshot=True,
snapshots_dir=tmp_dir,
timeout=3,
)
except infra.network.StartupSnapshotIsOld:
pass
def run(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_historical_query_range(network, args)
def run(args):
with infra.service_load.load() as load:
with infra.network.network(
args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb,
service_load=load,
) as network:
check = infra.checker.Checker()
network.start_and_open(args)
current_view = None
primary, current_view = network.find_primary()
# Number of nodes F to stop until network cannot make progress
nodes_to_stop = math.ceil(len(args.nodes) / 2)
if args.consensus == "BFT":
nodes_to_stop = math.ceil(len(args.nodes) / 3)
primary_is_known = True
for node_to_stop in range(nodes_to_stop):
primary, current_view = network.find_primary()
LOG.debug(
"Commit new transactions, primary:{}, current_view:{}".
format(primary.local_node_id, current_view))
with primary.client("user0") as c:
res = c.post(
"/app/log/private",
{
"id":
current_view,
"msg":
"This log is committed in view {}".format(
current_view),
},
)
check(res, result=True)
LOG.debug(
"Waiting for transaction to be committed by all nodes")
network.wait_for_all_nodes_to_commit(
tx_id=TxID(res.view, res.seqno))
try:
test_kill_primary_no_reqs(network, args)
except PrimaryNotFound:
if node_to_stop < nodes_to_stop - 1:
raise
else:
primary_is_known = False
assert not primary_is_known, "Primary is still known"
LOG.success("Test ended successfully.")
def run(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_host_process_launch(network, args)
network = test_host_process_launch_many(network, args)
def run_tls_san_checks(args):
with infra.network.network(
args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb,
) as network:
args.common_read_only_ledger_dir = None # Reset from previous test
network.start_and_open(args)
network.verify_service_certificate_validity_period(
args.initial_service_cert_validity_days)
LOG.info("Check SAN value in TLS certificate")
dummy_san = "*.dummy.com"
new_node = network.create_node(
infra.interfaces.HostSpec(
rpc_interfaces={
infra.interfaces.PRIMARY_RPC_INTERFACE:
infra.interfaces.
RPCInterface(endorsement=infra.interfaces.Endorsement(
authority=infra.interfaces.EndorsementAuthority.Node))
}))
args.subject_alt_names = [f"dNSName:{dummy_san}"]
network.join_node(new_node, args.package, args)
sans = infra.crypto.get_san_from_pem_cert(
new_node.get_tls_certificate_pem())
assert len(sans) == 1, "Expected exactly one SAN"
assert sans[0].value == dummy_san
LOG.info(
"A node started with no specified SAN defaults to public RPC host")
dummy_public_rpc_host = "123.123.123.123"
args.subject_alt_names = []
new_node = network.create_node(
infra.interfaces.HostSpec(
rpc_interfaces={
infra.interfaces.PRIMARY_RPC_INTERFACE:
infra.interfaces.RPCInterface(
public_host=dummy_public_rpc_host,
endorsement=infra.interfaces.Endorsement(
authority=infra.interfaces.EndorsementAuthority.
Node),
)
}))
network.join_node(new_node, args.package, args)
# Cannot trust the node here as client cannot authenticate dummy public IP in cert
with open(
os.path.join(network.common_dir,
f"{new_node.local_node_id}.pem"),
encoding="utf-8",
) as self_signed_cert:
sans = infra.crypto.get_san_from_pem_cert(self_signed_cert.read())
assert len(sans) == 1, "Expected exactly one SAN"
assert sans[0].value == ipaddress.ip_address(dummy_public_rpc_host)
def run_authn(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_jwt_auth(network, args)
network = test_multi_auth(network, args)
network = test_role_based_access(network, args)
def run(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_missing_signature_header(network, args)
network = test_corrupted_signature(network, args)
network = test_governance(network, args)
def run_content_types(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_content_types(network, args)
network = test_accept_header(network, args)
network = test_supported_methods(network, args)
network = test_unknown_path(network, args)
def run(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_primary(network, args)
network = test_network_node_info(network, args)
network = test_node_ids(network, args)
network = test_memory(network, args)
def run_manual(args):
with infra.network.network(
args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb
) as network:
network.start_and_open(args)
test_jwt_key_initial_refresh(network, args)
# Check that initial refresh also works on backups
primary, _ = network.find_primary()
primary.stop()
network.wait_for_new_primary(primary)
test_jwt_key_initial_refresh(network, args)
def js_gov(args):
with infra.network.network(
args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb
) as network:
network.start_and_open(args)
governance_js.test_proposal_validation(network, args)
governance_js.test_proposal_storage(network, args)
governance_js.test_proposal_withdrawal(network, args)
governance_js.test_ballot_storage(network, args)
governance_js.test_pure_proposals(network, args)
governance_js.test_proposals_with_votes(network, args)
governance_js.test_vote_failure_reporting(network, args)
governance_js.test_operator_proposals_and_votes(network, args)
governance_js.test_apply(network, args)
governance_js.test_set_constitution(network, args)
def run_limits(args):
if "v8" in args.package:
LOG.warning(
f"Skipping run_limits for {args.package} as heap and stack limits are not yet enforced"
)
# See https://github.com/microsoft/CCF/issues/3324
return
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_stack_size_limit(network, args)
network = test_heap_size_limit(network, args)
def run(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
test_verify_quotes(network, args)
test_add_node_with_bad_code(network, args)
# NB: Assumes the current nodes are still using args.package, so must run before test_proposal_invalidation
test_proposal_invalidation(network, args)
test_update_all_nodes(network, args)
# Run again at the end to confirm current nodes are acceptable
test_verify_quotes(network, args)
def run(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
network = test_module_import(network, args)
network = test_dynamic_module_import(network, args)
network = test_bytecode_cache(network, args)
network = test_app_bundle(network, args)
network = test_dynamic_endpoints(network, args)
if "v8" not in args.package:
# endpoint calls fail with "Cannot access \'logMap\' before init..."
# as if the const logMap wasn't preserved/captured
network = test_npm_app(network, args)
def run_to_destruction(args):
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
LOG.warning("About to issue transactions until destruction")
try:
wsm = 5000
while True:
LOG.info(f"Trying with writes scaled by {wsm}")
network = test(network,
args,
batch_size=10,
write_size_multiplier=wsm)
if wsm > 1000000:
LOG.error(
f"Run to destruction still hasn't caused exception with write sizes multiplied by {wsm}. Infinite loop, or not actually submitting?"
)
raise ValueError(wsm)
else:
wsm += 50000 # Grow very quickly, expect to fail on the second iteration
except Exception as e:
timeout = 10
LOG.info("Large write set caused an exception, as expected")
LOG.info(f"Exception was: {e}")
LOG.info(f"Polling for {timeout}s for node to terminate")
end_time = time.time() + timeout
while time.time() < end_time:
time.sleep(0.1)
exit_code = network.nodes[0].remote.remote.proc.poll()
if exit_code is not None:
LOG.info(f"Node terminated with exit code {exit_code}")
assert exit_code != 0
break
if time.time() > end_time:
raise TimeoutError(
f"Node took longer than {timeout}s to terminate") from e
network.ignore_errors_on_shutdown()
def run_auto(args):
with infra.network.network(
args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb
) as network:
network.start_and_open(args)
test_jwt_endpoint(network, args)
test_jwt_without_key_policy(network, args)
if args.enclave_type != "virtual":
test_jwt_with_sgx_key_policy(network, args)
test_jwt_with_sgx_key_filter(network, args)
test_jwt_key_auto_refresh(network, args)
# Check that auto refresh also works on backups
primary, _ = network.find_primary()
primary.stop()
network.wait_for_new_primary(primary)
test_jwt_key_auto_refresh(network, args)
def run(args):
with infra.network.network(
args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb
) as network:
for node in network.nodes:
node.curl = True
network.start_and_open(args)
primary, _ = network.find_primary()
uncommitted_ledger_dir, committed_ledger_dirs = list(primary.get_ledger())
cmd = [
"python",
args.ledger_tutorial,
*committed_ledger_dirs,
uncommitted_ledger_dir,
]
rc = infra.proc.ccall(*cmd).returncode
assert rc == 0, f"Failed to run tutorial script: {rc}"
def run_migration_tests(args):
if args.reconfiguration_type != "OneTransaction":
return
with infra.network.network(
args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb,
) as network:
network.start_and_open(args)
test_migration_2tx_reconfiguration(network, args)
primary, _ = network.find_primary()
new_node = network.nodes[-1]
ledger_paths = primary.remote.ledger_paths()
learner_id = new_node.node_id
check_2tx_ledger(ledger_paths, learner_id)
def run(args):
txs = app.LoggingTxs("user0")
with infra.network.network(
args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb,
txs=txs,
init_partitioner=True,
) as network:
network.start_and_open(args)
test_invalid_partitions(network, args)
test_partition_majority(network, args)
test_isolate_primary_from_one_backup(network, args)
test_new_joiner_helps_liveness(network, args)
for n in range(5):
test_isolate_and_reconnect_primary(network, args, iteration=n)
def run_2tx_reconfig_tests(args):
if not args.include_2tx_reconfig:
return
local_args = args
if args.reconfiguration_type != "TwoTransaction":
local_args.reconfiguration_type = "TwoTransaction"
with infra.network.network(
local_args.nodes,
local_args.binary_dir,
local_args.debug_nodes,
local_args.perf_nodes,
pdb=local_args.pdb,
init_partitioner=True,
) as network:
network.start_and_open(local_args)
test_learner_does_not_take_part(network, local_args)
def run(args):
txs = app.LoggingTxs("user0")
with infra.network.network(
args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb,
txs=txs,
) as network:
network.start_and_open(args)
test_version(network, args)
if args.consensus != "BFT":
test_join_straddling_primary_replacement(network, args)
test_node_replacement(network, args)
test_add_node_from_backup(network, args)
test_add_node(network, args)
test_add_node_on_other_curve(network, args)
test_retire_backup(network, args)
test_add_as_many_pending_nodes(network, args)
test_add_node(network, args)
test_retire_primary(network, args)
test_add_node_with_read_only_ledger(network, args)
test_add_node_from_snapshot(network, args)
test_add_node_from_snapshot(network, args, from_backup=True)
test_add_node_from_snapshot(network,
args,
copy_ledger_read_only=False)
test_node_filter(network, args)
test_retiring_nodes_emit_at_most_one_signature(network, args)
if args.reconfiguration_type == "TwoTransaction":
test_learner_catches_up(network, args)
test_service_config_endpoint(network, args)
test_node_certificates_validity_period(network, args)
test_add_node_invalid_validity_period(network, args)
def run_file_operations(args):
with tempfile.TemporaryDirectory() as tmp_dir:
txs = app.LoggingTxs("user0")
with infra.network.network(
args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb,
txs=txs,
) as network:
args.common_read_only_ledger_dir = tmp_dir
network.start_and_open(args)
test_save_committed_ledger_files(network, args)
test_parse_snapshot_file(network, args)
test_forced_ledger_chunk(network, args)
test_forced_snapshot(network, args)
primary, _ = network.find_primary()
network.stop_all_nodes()
test_split_ledger_on_stopped_network(primary, args)
def service_startups(args):
LOG.info("Starting service with insufficient number of recovery members")
args.initial_member_count = 2
args.initial_recovery_member_count = 0
args.initial_operator_count = 1
args.ledger_recovery_timeout = 5
with infra.network.network(args.nodes, args.binary_dir,
pdb=args.pdb) as network:
try:
network.start_and_open(args)
assert False, "Service cannot be opened with no recovery members"
except infra.proposal.ProposalNotAccepted:
primary, _ = network.find_primary()
network.consortium.check_for_service(
primary, infra.network.ServiceStatus.OPENING)
LOG.success(
"Service could not be opened with insufficient number of recovery members"
)
LOG.info(
"Starting service with a recovery operator member, a non-recovery operator member and a non-recovery non-operator member"
)
args.initial_member_count = 3
args.initial_recovery_member_count = 1
args.initial_operator_count = 2
with infra.network.network(args.nodes, args.binary_dir,
pdb=args.pdb) as network:
network.start_and_open(args)
LOG.info(
"Starting service with a recovery operator member, a recovery non-operator member and a non-recovery non-operator member"
)
args.initial_member_count = 3
args.initial_recovery_member_count = 2
args.initial_operator_count = 1
with infra.network.network(args.nodes, args.binary_dir,
pdb=args.pdb) as network:
network.start_and_open(args)
def run(args):
# This is deliberately 5, because the rest of the test depends on this
# to grow a prefix and allow just enough nodes to resume to reach the
# desired election result. Conversion to a general f isn't trivial.
hosts = ["local://localhost"] * 5
with infra.network.network(hosts,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_open(args)
primary, backups = network.find_nodes()
# Suspend three of the backups to prevent commit
backups[1].suspend()
backups[2].suspend()
backups[3].stop()
committable_txs = []
# Run some transactions that can't be committed now
with primary.client("user0") as uc:
for i in range(3):
committable_txs.append(
uc.post("/app/log/private", {
"id": 100 + i,
"msg": "Hello world"
}))
last_tx = committable_txs[-1]
sig_view, sig_seqno = last_tx.view, last_tx.seqno + 1
with backups[0].client() as bc:
wait_for_pending(bc, sig_view, sig_seqno)
# Suspend the final backup and run some transactions which only the partitioned
# primary hears, which should be discarded by the new primary
# NB: We can't guarantee that these will be discarded. Since we can't control
# what order the queued actions occur in after resuming, they may be appended
# before an election is called. They key assertion is that this primary is able
# to rejoin the network whatever happens, even when (in the usual case) they
# hold a suffix which has been discarded.
backups[0].suspend()
post_partition_txs = []
with primary.client("user0") as uc:
for i in range(3):
post_partition_txs.append(
uc.post("/app/log/private", {
"id": 100 + i,
"msg": "Hello world"
}))
# Sleep long enough that this primary should be instantly replaced when nodes wake
sleep_time = 2 * args.election_timeout_ms / 1000
LOG.info(f"Sleeping {sleep_time}s")
time.sleep(sleep_time)
# Suspend the primary, resume other backups
primary.suspend()
backups[0].resume()
backups[1].resume()
backups[2].resume()
new_primary, _ = network.wait_for_new_primary(primary,
timeout_multiplier=10)
with new_primary.client("user0") as uc:
# Check that uncommitted but committable suffix is preserved
check_commit = infra.checker.Checker(uc)
for tx in committable_txs:
check_commit(tx)
# Check that new transactions can be committed
with new_primary.client("user0") as uc:
for i in range(3):
r = uc.post("/app/log/private", {
"id": 100 + i,
"msg": "Hello world"
})
assert r.status_code == 200
uc.wait_for_commit(r)
# Resume original primary, check that they rejoin correctly, including new transactions
primary.resume()
network.wait_for_node_commit_sync(timeout=16)
