def test_all_nodes_cert_renewal(network, args, valid_from=None): primary, _ = network.find_primary() valid_from = valid_from or datetime.now() validity_period_days = args.maximum_node_certificate_validity_days self_signed_node_certs_before = {} for node in network.get_joined_nodes(): # Note: GET /node/self_signed_certificate endpoint was added after 2.0.0-r6 if node.version_after("ccf-2.0.0-rc6"): self_signed_node_certs_before[ node.local_node_id] = node.retrieve_self_signed_cert() network.consortium.set_all_nodes_certificate_validity( primary, valid_from=valid_from, validity_period_days=validity_period_days, ) # Node certificates are updated on global commit hook network.wait_for_all_nodes_to_commit(primary) for node in network.get_joined_nodes(): node.set_certificate_validity_period(valid_from, validity_period_days) if node.version_after("ccf-2.0.0-rc6"): assert ( self_signed_node_certs_before[node.local_node_id] != node.retrieve_self_signed_cert() ), f"Self-signed node certificate for node {node.local_node_id} was not renewed"
def test_invalid_partitions(network, args): nodes = network.get_joined_nodes() try: network.partitioner.partition( [nodes[0], nodes[2]], [nodes[1], nodes[2]], ) assert False, "Node should not appear in two or more partitions" except ValueError: pass try: network.partitioner.partition() assert False, "At least one partition should be specified" except ValueError: pass try: invalid_local_node_id = -1 new_node = infra.node.Node(invalid_local_node_id, "local://localhost") network.partitioner.partition([new_node]) assert False, "All nodes should belong to network" except ValueError: pass return network
def test_kill_primary_no_reqs(network, args): old_primary, _ = network.find_primary_and_any_backup() old_primary.stop() new_primary, _ = network.wait_for_new_primary(old_primary) # Verify that the TxID reported just after an election is valid # Note that the first TxID read after an election may be of a signature # Tx (time-based signature generation) in the new term rather than the # last entry in the previous term for node in network.get_joined_nodes(): with node.client() as c: r = c.get("/node/network") c.wait_for_commit(r) # Also verify that reported last ack time are as expected r = c.get("/node/consensus") acks = r.body.json()["details"]["acks"] for ack in acks.values(): if node is new_primary: assert (ack["last_received_ms"] < network.args.election_timeout_ms), acks else: assert ( ack["last_received_ms"] == 0 ), f"Backup {node.local_node_id} should report time of last acks of 0: {acks}" return network
def check(network, args, *nargs, **kwargs): primary, _ = network.find_primary() with primary.client( network.consortium.get_any_active_member().local_id) as c: r = c.post( "/gov/query", { "text": """tables = ... trusted_nodes_count = 0 tables["public:ccf.gov.nodes.info"]:foreach(function(node_id, details) if details["status"] == "TRUSTED" then trusted_nodes_count = trusted_nodes_count + 1 end end) return trusted_nodes_count """ }, ) trusted_nodes_count = r.body.json() running_nodes_count = len(network.get_joined_nodes()) would_leave_nodes_count = running_nodes_count - nodes_to_kill_count minimum_nodes_to_run_count = ceil((trusted_nodes_count + 1) / 2) if args.consensus == "cft" and (would_leave_nodes_count < minimum_nodes_to_run_count): raise TestRequirementsNotMet( f"Cannot kill {nodes_to_kill_count} node(s) as the network would not be able to make progress" f" (would leave {would_leave_nodes_count} nodes but requires {minimum_nodes_to_run_count} nodes to make progress) " )
def run(args): txs = app.LoggingTxs() with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs, ) as network: network.start_and_join(args) test_add_node_from_backup(network, args) test_add_node(network, args) test_retire_backup(network, args) test_add_as_many_pending_nodes(network, args) test_add_node(network, args) test_retire_primary(network, args) test_add_node_from_snapshot(network, args) test_add_node_from_snapshot(network, args, from_backup=True) test_add_node_from_snapshot(network, args, copy_ledger_read_only=False) latest_node_log = network.get_joined_nodes()[-1].remote.log_path() with open(latest_node_log, "r+") as log: assert any( "No snapshot found: Node will replay all historical transactions" in l for l in log.readlines() ), "New nodes shouldn't join from snapshot if snapshot evidence cannot be verified" test_node_filter(network, args)
def wait_for_reconfiguration_to_complete(network, timeout=10): max_num_configs = 0 max_rid = 0 all_same_rid = False end_time = time.time() + timeout while max_num_configs > 1 or not all_same_rid: max_num_configs = 0 all_same_rid = True for node in network.get_joined_nodes(): with node.client(self_signed_ok=True) as c: try: r = c.get("/node/consensus") rj = r.body.json() cfgs = rj["details"]["configs"] num_configs = len(cfgs) max_num_configs = max(max_num_configs, num_configs) if num_configs == 1 and cfgs[0]["rid"] != max_rid: max_rid = max(max_rid, cfgs[0]["rid"]) all_same_rid = False except Exception as ex: # OK, retiring node may be gone or a joining node may not be ready yet LOG.info(f"expected RPC failure because of: {ex}") time.sleep(0.5) LOG.info(f"max num configs: {max_num_configs}, max rid: {max_rid}") if time.time() > end_time: raise Exception("Reconfiguration did not complete in time")
def test_version(network, args): nodes = network.get_joined_nodes() for node in nodes: with node.client() as c: r = c.get("/node/version") assert r.body.json()["ccf_version"] == args.ccf_version
def run(args): txs = app.LoggingTxs() with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs, ) as network: network.start_and_join(args) test_add_node_from_backup(network, args) test_add_node(network, args) test_add_node_untrusted_code(network, args) test_retire_backup(network, args) test_add_as_many_pending_nodes(network, args) test_add_node(network, args) test_retire_primary(network, args) if args.snapshot_tx_interval is not None: test_add_node_from_snapshot(network, args, copy_ledger_read_only=True) test_add_node_from_snapshot(network, args, copy_ledger_read_only=False) errors, _ = network.get_joined_nodes()[-1].stop() if not any( "No snapshot found: Node will request all historical transactions" in s for s in errors): raise ValueError( "New node shouldn't join from snapshot if snapshot cannot be verified" )
def test_view_history(network, args): if args.consensus == "pbft": # This appears to work in PBFT, but it is unacceptably slow: # - Each /tx request is a write, with a non-trivial roundtrip response time # - Since each read (eg - /tx and /commit) has produced writes and a unique tx ID, # there are too many IDs to test exhaustively # We could rectify this by making this test non-exhaustive (bisecting for view changes, # sampling within a view), but for now it is exhaustive and Raft-only LOG.warning("Skipping view reconstruction in PBFT") return network check = infra.checker.Checker() for node in network.get_joined_nodes(): with node.client("user0") as c: r = c.get("/node/commit") check(c) commit_view = r.body["view"] commit_seqno = r.body["seqno"] # Retrieve status for all possible Tx IDs seqno_to_views = {} for seqno in range(1, commit_seqno + 1): views = [] for view in range(1, commit_view + 1): r = c.get(f"/node/tx?view={view}&seqno={seqno}") check(r) status = TxStatus(r.body["status"]) if status == TxStatus.Committed: views.append(view) seqno_to_views[seqno] = views # Check we have exactly one Tx ID for each seqno txs_ok = True for seqno, views in seqno_to_views.items(): if len(views) != 1: txs_ok = False LOG.error( f"Node {node.node_id}: Found {len(views)} committed Tx IDs for seqno {seqno}" ) tx_ids_condensed = ", ".join( " OR ".join(f"{view}.{seqno}" for view in views or ["UNKNOWN"]) for seqno, views in seqno_to_views.items()) if txs_ok: LOG.success( f"Node {node.node_id}: Found a valid sequence of Tx IDs:\n{tx_ids_condensed}" ) else: LOG.error( f"Node {node.node_id}: Invalid sequence of Tx IDs:\n{tx_ids_condensed}" ) raise RuntimeError( f"Node {node.node_id}: Incomplete or inconsistent view history" ) return network
def run(args): # Three nodes minimum to make sure that the raft network can still make progress # if one node stops hosts = ["localhost"] * (4 if args.consensus == "bft" else 3) with infra.network.network(hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: check = infra.checker.Checker() network.start_and_join(args) current_view = None # Number of nodes F to stop until network cannot make progress nodes_to_stop = math.ceil(len(hosts) / 2) if args.consensus == "bft": nodes_to_stop = math.ceil(len(hosts) / 3) primary_is_known = True for node_to_stop in range(nodes_to_stop): # Note that for the first iteration, the primary is known in advance anyway LOG.debug("Find freshly elected primary") # After a view change in pbft, finding the new primary takes longer primary, current_view = network.find_primary( timeout=(30 if args.consensus == "bft" else 3)) LOG.debug( "Commit new transactions, primary:{}, current_view:{}".format( primary.node_id, current_view)) with primary.client("user0") as c: res = c.post( "/app/log/private", { "id": current_view, "msg": "This log is committed in view {}".format( current_view), }, ) check(res, result=True) seqno = res.seqno LOG.debug("Waiting for transaction to be committed by all nodes") wait_for_seqno_to_commit(seqno, current_view, network.get_joined_nodes()) try: test_kill_primary(network, args) except PrimaryNotFound: if node_to_stop < nodes_to_stop - 1: raise else: primary_is_known = False assert not primary_is_known, "Primary is still known" LOG.success("Test ended successfully.")
def test_quote(network, args): primary, _ = network.find_nodes() with primary.client() as c: oed = subprocess.run( [ os.path.join(args.oe_binary, "oesign"), "dump", "-e", infra.path.build_lib_path(args.package, args.enclave_type), ], capture_output=True, check=True, ) lines = [ line for line in oed.stdout.decode().split(os.linesep) if line.startswith("mrenclave=") ] expected_mrenclave = lines[0].strip().split("=")[1] r = c.get("/node/quotes/self") primary_quote_info = r.body.json() assert primary_quote_info["node_id"] == 0 primary_mrenclave = primary_quote_info["mrenclave"] assert primary_mrenclave == expected_mrenclave, ( primary_mrenclave, expected_mrenclave, ) r = c.get("/node/quotes") quotes = r.body.json()["quotes"] assert len(quotes) == len(network.get_joined_nodes()) for quote in quotes: mrenclave = quote["mrenclave"] assert mrenclave == expected_mrenclave, (mrenclave, expected_mrenclave) quote_path = os.path.join(network.common_dir, f"quote{quote['node_id']}") endorsements_path = os.path.join( network.common_dir, f"endorsements{quote['node_id']}") with open(quote_path, "wb") as q: q.write(bytes.fromhex(quote["raw"])) with open(endorsements_path, "wb") as e: e.write(bytes.fromhex(quote["endorsements"])) cafile = os.path.join(network.common_dir, "networkcert.pem") assert (infra.proc.ccall( "verify_quote.sh", f"https://{primary.pubhost}:{primary.pubport}", "--cacert", f"{cafile}", log_output=True, ).returncode == 0 ), f"Quote verification for node {quote['node_id']} failed" return network
def test_each_node_cert_renewal(network, args): primary, _ = network.find_primary() now = datetime.now() validity_period_allowed = args.max_allowed_node_cert_validity_days - 1 validity_period_forbidden = args.max_allowed_node_cert_validity_days + 1 test_vectors = [ (now, validity_period_allowed, None), (now, None, None), # Omit validity period (deduced from service configuration) (now, -1, infra.proposal.ProposalNotCreated), (now, validity_period_forbidden, infra.proposal.ProposalNotAccepted), ] for (valid_from, validity_period_days, expected_exception) in test_vectors: for node in network.get_joined_nodes(): with node.client() as c: c.get("/node/network/nodes") node_cert_tls_before = node.get_tls_certificate_pem() assert (infra.crypto.compute_public_key_der_hash_hex_from_pem( node_cert_tls_before) == node.node_id) try: valid_from_x509 = str( infra.crypto.datetime_to_X509time(valid_from)) network.consortium.set_node_certificate_validity( primary, node, valid_from=valid_from_x509, validity_period_days=validity_period_days, ) node.set_certificate_validity_period( valid_from_x509, validity_period_days or args.max_allowed_node_cert_validity_days, ) except Exception as e: assert isinstance(e, expected_exception) continue else: assert (expected_exception is None), "Proposal should have not succeeded" node_cert_tls_after = node.get_tls_certificate_pem() assert ( node_cert_tls_before != node_cert_tls_after ), f"Node {node.local_node_id} certificate was not renewed" node.verify_certificate_validity_period() LOG.info( f"Certificate for node {node.local_node_id} has successfully been renewed" ) # Long-connected client is still connected after certificate renewal c.get("/node/network/nodes") return network
def node_configs(network): configs = {} for node in network.get_joined_nodes(): try: with node.client() as nc: configs[node.node_id] = nc.get("/node/config").body.json() except Exception: pass return configs
def mem_stats(network): mem = {} for node in network.get_joined_nodes(): try: with node.client() as c: r = c.get("/node/memory", 0.1) mem[node.node_id] = r.body.json() except Exception: pass return mem
def test_quote(network, args): if args.enclave_type == "virtual": LOG.warning("Quote test can only run in real enclaves, skipping") return network primary, _ = network.find_nodes() with primary.client() as c: oed = subprocess.run( [ os.path.join(args.oe_binary, "oesign"), "dump", "-e", infra.path.build_lib_path(args.package, args.enclave_type), ], capture_output=True, check=True, ) lines = [ line for line in oed.stdout.decode().split(os.linesep) if line.startswith("mrenclave=") ] expected_mrenclave = lines[0].strip().split("=")[1] r = c.get("/node/quotes/self") primary_quote_info = r.body.json() assert primary_quote_info["node_id"] == primary.node_id primary_mrenclave = primary_quote_info["mrenclave"] assert primary_mrenclave == expected_mrenclave, ( primary_mrenclave, expected_mrenclave, ) r = c.get("/node/quotes") quotes = r.body.json()["quotes"] assert len(quotes) == len(network.get_joined_nodes()) for quote in quotes: mrenclave = quote["mrenclave"] assert mrenclave == expected_mrenclave, (mrenclave, expected_mrenclave) cafile = os.path.join(network.common_dir, "service_cert.pem") assert ( infra.proc.ccall( "verify_quote.sh", f"https://{primary.get_public_rpc_host()}:{primary.get_public_rpc_port()}", "--cacert", f"{cafile}", log_output=True, ).returncode == 0 ), f"Quote verification for node {quote['node_id']} failed" return network
def wrapper(*args, **kwargs): network = args[0] infra.e2e_args = vars(args[1]) snapshot_tx_interval = infra.e2e_args.get("snapshot_tx_interval") if snapshot_tx_interval is not None: issue_historical_queries_with_snapshot( network, int(snapshot_tx_interval)) network = func(*args, **kwargs) # Only verify entries on node just added network.txs.verify(node=network.get_joined_nodes()[-1]) return network
def test_version(network, args): if args.ccf_version is None: LOG.warning( "Skipping network version check as no expected version is specified" ) return nodes = network.get_joined_nodes() for node in nodes: with node.client() as c: r = c.get("/node/version") assert r.body.json()["ccf_version"] == args.ccf_version
def test_all_nodes_cert_renewal(network, args): primary, _ = network.find_primary() valid_from = str(infra.crypto.datetime_to_X509time(datetime.now())) validity_period_days = args.maximum_node_certificate_validity_days network.consortium.set_all_nodes_certificate_validity( primary, valid_from=valid_from, validity_period_days=validity_period_days, ) for node in network.get_joined_nodes(): node.set_certificate_validity_period(valid_from, validity_period_days)
def test_kill_primary(network, args): primary, _ = network.find_primary_and_any_backup() primary.stop() network.wait_for_new_primary(primary) # Verify that the TxID reported just after an election is valid # Note that the first TxID read after an election may be of a signature # Tx (time-based signature generation) in the new term rather than the # last entry in the previous term for node in network.get_joined_nodes(): with node.client() as c: r = c.get("/node/network") c.wait_for_commit(r) return network
def test_node_ids(network, args): nodes = network.get_joined_nodes() for node in nodes: for _, interface in node.host.rpc_interfaces.items(): with node.client() as c: r = c.get( f"/node/network/nodes?host={interface.public_host}&port={interface.public_port}" ) assert r.status_code == http.HTTPStatus.OK.value info = r.body.json()["nodes"] assert len(info) == 1 assert info[0]["node_id"] == node.node_id assert info[0]["status"] == NodeStatus.TRUSTED.value assert len(info[0]["rpc_interfaces"]) == len( node.host.rpc_interfaces) return network
def test_verify_quotes(network, args): if args.enclave_type == "virtual": LOG.warning("Skipping quote test with virtual enclave") return network for node in network.get_joined_nodes(): LOG.info(f"Verifying quote for node {node.node_id}") cafile = os.path.join(network.common_dir, "networkcert.pem") assert (infra.proc.ccall( "verify_quote.sh", f"https://{node.pubhost}:{node.pubport}", "--cacert", f"{cafile}", log_output=True, ).returncode == 0 ), f"Quote verification for node {node.node_id} failed" return network
def test_new_service( network, args, install_path, binary_dir, library_dir, version, cycle_existing_nodes=False, ): LOG.info("Update constitution") primary, _ = network.find_primary() new_constitution = get_new_constitution_for_install(args, install_path) network.consortium.set_constitution(primary, new_constitution) # Note: Changes to constitution between versions should be tested here LOG.info(f"Add node to new service [cycle nodes: {cycle_existing_nodes}]") nodes_to_cycle = network.get_joined_nodes() if cycle_existing_nodes else [] nodes_to_add_count = len(nodes_to_cycle) if cycle_existing_nodes else 1 for _ in range(0, nodes_to_add_count): new_node = network.create_node( "local://localhost", binary_dir=binary_dir, library_dir=library_dir, version=version, ) network.join_node(new_node, args.package, args) network.trust_node(new_node, args) new_node.verify_certificate_validity_period( expected_validity_period_days=DEFAULT_NODE_CERTIFICATE_VALIDITY_DAYS ) for node in nodes_to_cycle: network.retire_node(primary, node) if primary == node: primary, _ = network.wait_for_new_primary(primary) node.stop() test_all_nodes_cert_renewal(network, args) LOG.info("Apply transactions to new nodes only") issue_activity_on_live_service(network, args) test_random_receipts(network, args, lts=True)
def test_recover_service(network, args, from_snapshot=False): network.save_service_identity(args) old_primary, _ = network.find_primary() snapshots_dir = None if from_snapshot: snapshots_dir = network.get_committed_snapshots(old_primary) # Start health watcher and stop nodes one by one until a recovery has to be staged watcher = infra.health_watcher.NetworkHealthWatcher(network, args, verbose=True) watcher.start() for node in network.get_joined_nodes(): time.sleep(args.election_timeout_ms / 1000) node.stop() watcher.wait_for_recovery() # Stop remaining nodes network.stop_all_nodes() current_ledger_dir, committed_ledger_dirs = old_primary.get_ledger() recovered_network = infra.network.Network( args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, existing_network=network, ) recovered_network.start_in_recovery( args, ledger_dir=current_ledger_dir, committed_ledger_dirs=committed_ledger_dirs, snapshots_dir=snapshots_dir, ) recovered_network.recover(args) return recovered_network
def test_verify_quotes(network, args): if args.enclave_type == "virtual": LOG.warning("Skipping quote test with virtual enclave") return network LOG.info("Check the network is stable") primary, _ = network.find_primary() check_can_progress(primary) for node in network.get_joined_nodes(): LOG.info(f"Verifying quote for node {node.node_id}") cafile = os.path.join(network.common_dir, "service_cert.pem") assert (infra.proc.ccall( "verify_quote.sh", f"https://{node.get_public_rpc_host()}:{node.get_public_rpc_port()}", "--cacert", f"{cafile}", log_output=True, ).returncode == 0 ), f"Quote verification for node {node.node_id} failed" return network
def test_node_replacement(network, args): primary, backups = network.find_nodes() nodes = network.get_joined_nodes() node_to_replace = backups[-1] f = infra.e2e_args.max_f(args, len(nodes)) f_backups = backups[:f] # Retire one node network.consortium.retire_node(primary, node_to_replace) node_to_replace.stop() network.nodes.remove(node_to_replace) check_can_progress(primary) # Add in a node using the same address replacement_node = network.create_and_trust_node( args.package, f"local://{node_to_replace.host}:{node_to_replace.rpc_port}", args, node_port=node_to_replace.node_port, from_snapshot=False, ) assert replacement_node.node_id != node_to_replace.node_id assert replacement_node.host == node_to_replace.host assert replacement_node.node_port == node_to_replace.node_port assert replacement_node.rpc_port == node_to_replace.rpc_port LOG.info( f"Stopping {len(f_backups)} other nodes to make progress depend on the replacement" ) for other_backup in f_backups: other_backup.suspend() # Confirm the network can make progress check_can_progress(primary) for other_backup in f_backups: other_backup.resume() return network
def test_ack_state_digest_update(network, args): for node in network.get_joined_nodes(): network.consortium.get_any_active_member().update_ack_state_digest(node) return network
def check(network, args, *nargs, **kwargs): running_nodes = len(network.get_joined_nodes()) if running_nodes < n: raise TestRequirementsNotMet( f"Too few nodes. Only have {running_nodes}, requires at least {n}" )
def run(args): hosts = ["localhost", "localhost", "localhost"] LOG.info(f"setting seed to {args.seed}") random.seed(args.seed) txs = app.LoggingTxs() with infra.network.network( hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs ) as network: network.start_and_join(args) original_nodes = network.get_joined_nodes() view_info = {} suspend.update_view_info(network, view_info) app.test_run_txs(network=network, args=args, num_txs=TOTAL_REQUESTS) suspend.update_view_info(network, view_info) nodes_to_kill = [network.find_any_backup()] nodes_to_keep = [n for n in original_nodes if n not in nodes_to_kill] # check that a new node can catch up after all the requests late_joiner = network.create_and_trust_node(args.package, "localhost", args) nodes_to_keep.append(late_joiner) # some requests to be processed while the late joiner catches up # (no strict checking that these requests are actually being processed simultaneously with the node catchup) app.test_run_txs( network=network, args=args, num_txs=int(TOTAL_REQUESTS / 2), nodes=original_nodes, # doesn't contain late joiner verify=False, # will try to verify for late joiner and it might not be ready yet ) suspend.wait_for_late_joiner(original_nodes[0], late_joiner) # kill the old node(s) and ensure we are still making progress for backup_to_retire in nodes_to_kill: LOG.success(f"Stopping node {backup_to_retire.node_id}") backup_to_retire.stop() # check nodes are ok after we killed one off app.test_run_txs( network=network, args=args, nodes=nodes_to_keep, num_txs=len(nodes_to_keep), timeout=30, ignore_failures=True, # in the event of an early view change due to the late joiner this might # take longer than usual to complete and we don't want the test to break here ) suspend.test_suspend_nodes(network, args, nodes_to_keep) # run txs while nodes get suspended app.test_run_txs( network=network, args=args, num_txs=4 * TOTAL_REQUESTS, timeout=30, ignore_failures=True, # in the event of an early view change due to the late joiner this might # take longer than usual to complete and we don't want the test to break here ) suspend.update_view_info(network, view_info) # check nodes have resumed normal execution before shutting down app.test_run_txs(network=network, args=args, num_txs=len(nodes_to_keep)) # we have asserted that all nodes are caught up # assert that view changes actually did occur assert len(view_info) > 1 LOG.success("----------- views and primaries recorded -----------") for view, primary in view_info.items(): LOG.success(f"view {view} - primary {primary}")
def test_view_history(network, args): if args.consensus == "bft": # This appears to work in BFT, but it is unacceptably slow: # - Each /tx request is a write, with a non-trivial roundtrip response time # - Since each read (eg - /tx and /commit) has produced writes and a unique tx ID, # there are too many IDs to test exhaustively # We could rectify this by making this test non-exhaustive (bisecting for view changes, # sampling within a view), but for now it is exhaustive and Raft-only LOG.warning("Skipping view reconstruction in BFT") return network check = infra.checker.Checker() previous_node = None previous_tx_ids = "" for node in network.get_joined_nodes(): with node.client("user0") as c: r = c.get("/node/commit") check(c) commit_tx_id = TxID.from_str(r.body.json()["transaction_id"]) # Retrieve status for all possible Tx IDs seqno_to_views = {} for seqno in range(1, commit_tx_id.seqno + 1): views = [] for view in range(1, commit_tx_id.view + 1): r = c.get(f"/node/tx?transaction_id={view}.{seqno}", log_capture=[]) check(r) status = TxStatus(r.body.json()["status"]) if status == TxStatus.Committed: views.append(view) seqno_to_views[seqno] = views # Check we have exactly one Tx ID for each seqno txs_ok = True for seqno, views in seqno_to_views.items(): if len(views) != 1: txs_ok = False LOG.error( f"Node {node.node_id}: Found {len(views)} committed Tx IDs for seqno {seqno}" ) tx_ids_condensed = ", ".join( " OR ".join(f"{view}.{seqno}" for view in views or ["UNKNOWN"]) for seqno, views in seqno_to_views.items() ) if txs_ok: LOG.success( f"Node {node.node_id}: Found a valid sequence of Tx IDs:\n{tx_ids_condensed}" ) else: LOG.error( f"Node {node.node_id}: Invalid sequence of Tx IDs:\n{tx_ids_condensed}" ) raise RuntimeError( f"Node {node.node_id}: Incomplete or inconsistent view history" ) # Compare view history between nodes if previous_tx_ids: # Some nodes may have a slightly longer view history so only compare the common prefix min_tx_ids_len = min(len(previous_tx_ids), len(tx_ids_condensed)) assert ( tx_ids_condensed[:min_tx_ids_len] == previous_tx_ids[:min_tx_ids_len] ), f"Tx IDs don't match between node {node.node_id} and node {previous_node.node_id}: {tx_ids_condensed[:min_tx_ids_len]} and {previous_tx_ids[:min_tx_ids_len]}" previous_tx_ids = tx_ids_condensed previous_node = node return network
def test_new_service( network, args, install_path, binary_dir, library_dir, version, cycle_existing_nodes=False, ): LOG.info("Update constitution") primary, _ = network.find_primary() new_constitution = get_new_constitution_for_install(args, install_path) network.consortium.set_constitution(primary, new_constitution) all_nodes = network.get_joined_nodes() # Note: Changes to constitution between versions should be tested here LOG.info(f"Add node to new service [cycle nodes: {cycle_existing_nodes}]") nodes_to_cycle = network.get_joined_nodes() if cycle_existing_nodes else [] nodes_to_add_count = len(nodes_to_cycle) if cycle_existing_nodes else 1 # Pre-2.0 nodes require X509 time format valid_from = str(infra.crypto.datetime_to_X509time( datetime.datetime.now())) for _ in range(0, nodes_to_add_count): new_node = network.create_node( "local://localhost", binary_dir=binary_dir, library_dir=library_dir, version=version, ) network.join_node(new_node, args.package, args) network.trust_node( new_node, args, valid_from=valid_from, ) new_node.verify_certificate_validity_period( expected_validity_period_days=DEFAULT_NODE_CERTIFICATE_VALIDITY_DAYS ) all_nodes.append(new_node) for node in nodes_to_cycle: network.retire_node(primary, node) if primary == node: primary, _ = network.wait_for_new_primary(primary) node.stop() test_all_nodes_cert_renewal(network, args, valid_from=valid_from) test_service_cert_renewal(network, args, valid_from=valid_from) LOG.info("Waiting for retired nodes to be automatically removed") for node in all_nodes: network.wait_for_node_in_store( primary, node.node_id, node_status=ccf.ledger.NodeStatus.TRUSTED if node.is_joined() else None, ) if args.check_2tx_reconfig_migration: test_migration_2tx_reconfiguration( network, args, initial_is_1tx=False, # Reconfiguration type added in 2.x binary_dir=binary_dir, library_dir=library_dir, version=version, valid_from=valid_from, ) LOG.info("Apply transactions to new nodes only") issue_activity_on_live_service(network, args) test_random_receipts(network, args, lts=True)