def run(args): # Keep track of governance operations that happened in the test governance_operations = set() with infra.network.network(args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_join(args) primary, _ = network.find_primary() ledger_directories = primary.remote.ledger_paths() LOG.info("Add new member proposal (implicit vote)") ( new_member_proposal, _, careful_vote, ) = network.consortium.generate_and_propose_new_member( primary, curve=infra.network.EllipticCurve.secp256r1) member = network.consortium.get_member_by_local_id( new_member_proposal.proposer_id) governance_operations.add( (new_member_proposal.proposal_id, member.service_id, "propose")) LOG.info("2/3 members accept the proposal") p = network.consortium.vote_using_majority(primary, new_member_proposal, careful_vote) for voter in p.voters: governance_operations.add((p.proposal_id, voter, "vote")) assert new_member_proposal.state == infra.proposal.ProposalState.ACCEPTED LOG.info("Create new proposal but withdraw it before it is accepted") new_member_proposal, _, _ = network.consortium.generate_and_propose_new_member( primary, curve=infra.network.EllipticCurve.secp256r1) member = network.consortium.get_member_by_local_id( new_member_proposal.proposer_id) governance_operations.add( (new_member_proposal.proposal_id, member.service_id, "propose")) with primary.client() as c: response = network.consortium.get_member_by_local_id( new_member_proposal.proposer_id).withdraw( primary, new_member_proposal) infra.checker.Checker(c)(response) assert response.status_code == http.HTTPStatus.OK.value assert response.body.json()["state"] == ProposalState.WITHDRAWN.value member = network.consortium.get_member_by_local_id( new_member_proposal.proposer_id) governance_operations.add( (new_member_proposal.proposal_id, member.service_id, "withdraw")) # Force ledger flush of all transactions so far network.get_latest_ledger_public_state() ledger = ccf.ledger.Ledger(ledger_directories) check_operations(ledger, governance_operations) test_ledger_is_readable(network, args) test_tables_doc(network, args)
def test_service_principals(network, args): node = network.find_node_by_role() principal_id = "0xdeadbeef" # Initially, there is nothing in this table latest_public_tables, _ = network.get_latest_ledger_public_state() assert "public:ccf.gov.service_principals" not in latest_public_tables # Create and accept a proposal which populates an entry in this table principal_data = {"name": "Bob", "roles": ["Fireman", "Zookeeper"]} proposal = { "actions": [{ "name": "set_service_principal", "args": { "id": principal_id, "data": principal_data }, }] } ballot = { "ballot": "export function vote(proposal, proposer_id) { return true; }" } proposal = network.consortium.get_any_active_member().propose( node, proposal) network.consortium.vote_using_majority(node, proposal, ballot) # Confirm it can be read latest_public_tables, _ = network.get_latest_ledger_public_state() assert (json.loads( latest_public_tables["public:ccf.gov.service_principals"][ principal_id.encode()]) == principal_data) # Create and accept a proposal which removes an entry from this table proposal = { "actions": [{ "name": "remove_service_principal", "args": { "id": principal_id } }] } proposal = network.consortium.get_any_active_member().propose( node, proposal) network.consortium.vote_using_majority(node, proposal, ballot) # Confirm it is gone latest_public_tables, _ = network.get_latest_ledger_public_state() assert (principal_id.encode() not in latest_public_tables["public:ccf.gov.service_principals"]) return network
def get_current_nodes_table(network): tables, _ = network.get_latest_ledger_public_state() tn = "public:ccf.gov.nodes.info" r = {} for nid, info in tables[tn].items(): r[nid.decode()] = json.loads(info) return r
def test_read_ledger_utility(network, args): def fmt_str(data: bytes) -> str: return data.decode() format_rule = [(".*records.*", {"key": fmt_str, "value": fmt_str})] # Issue at least one transaction to see how it is read in the ledger network.txs.issue(network, number_txs=1) network.get_latest_ledger_public_state() primary, backups = network.find_nodes() for node in (primary, *backups): ledger_dirs = node.remote.ledger_paths() assert ccf.read_ledger.run(ledger_dirs, tables_format_rules=format_rule) return network
def test_retiring_nodes_emit_at_most_one_signature(network, args): primary, _ = network.find_primary() # Force ledger flush of all transactions so far network.get_latest_ledger_public_state() ledger = ccf.ledger.Ledger(primary.remote.ledger_paths()) retiring_nodes = set() retired_nodes = set() for chunk in ledger: for tr in chunk: tables = tr.get_public_domain().get_tables() if ccf.ledger.NODES_TABLE_NAME in tables: nodes = tables[ccf.ledger.NODES_TABLE_NAME] for nid, info_ in nodes.items(): if info_ is None: # Node was removed continue info = json.loads(info_) if info["status"] == "Retired": retiring_nodes.add(nid) if ccf.ledger.SIGNATURE_TX_TABLE_NAME in tables: sigs = tables[ccf.ledger.SIGNATURE_TX_TABLE_NAME] assert len(sigs) == 1, sigs.keys() (sig_,) = sigs.values() sig = json.loads(sig_) assert ( sig["node"] not in retired_nodes ), f"Unexpected signature from {sig['node']}" retired_nodes |= retiring_nodes retiring_nodes = set() assert not retiring_nodes, (retiring_nodes, retired_nodes) LOG.info("{} nodes retired throughout test", len(retired_nodes)) wait_for_reconfiguration_to_complete(network) return network
def check_kv_jwt_key_matches(network, kid, cert_pem): latest_public_state, _ = network.get_latest_ledger_public_state() latest_jwt_signing_key = latest_public_state[ "public:ccf.gov.jwt.public_signing_keys" ] if cert_pem is None: assert kid.encode() not in latest_jwt_signing_key else: stored_cert = infra.crypto.cert_der_to_pem(latest_jwt_signing_key[kid.encode()]) assert infra.crypto.are_certs_equal( cert_pem, stored_cert ), "input cert is not equal to stored cert"
def test_read_ledger_utility(network, args): def fmt_str(data: bytes) -> str: return data.decode() format_rule = [(".*records.*", {"key": fmt_str, "value": fmt_str})] network.txs.issue(network, number_txs=args.snapshot_tx_interval) network.get_latest_ledger_public_state() primary, backups = network.find_nodes() for node in (primary, *backups): ledger_dirs = node.remote.ledger_paths() assert ccf.read_ledger.run(paths=ledger_dirs, tables_format_rules=format_rule) snapshot_dir = network.get_committed_snapshots(primary) assert ccf.read_ledger.run( paths=[os.path.join(snapshot_dir, os.listdir(snapshot_dir)[-1])], is_snapshot=True, tables_format_rules=format_rule, ) return network
def test_member_data(network, args): assert args.initial_operator_count > 0 latest_public_tables, _ = network.get_latest_ledger_public_state() members_info = latest_public_tables["public:ccf.gov.members.info"] md_count = 0 for member in network.get_members(): stored_member_info = json.loads(members_info[member.service_id.encode()]) if member.member_data: assert ( stored_member_info["member_data"] == member.member_data ), f'stored member data "{stored_member_info["member_data"]}" != expected "{member.member_data} "' md_count += 1 else: assert "member_data" not in stored_member_info assert md_count == args.initial_operator_count return network
def run_code_upgrade_from( args, from_install_path, to_install_path, from_version=None, to_version=None, from_container_image=None, ): from_binary_dir, from_library_dir = get_bin_and_lib_dirs_for_install_path( from_install_path) to_binary_dir, to_library_dir = get_bin_and_lib_dirs_for_install_path( to_install_path) set_js_args(args, from_install_path, to_install_path) jwt_issuer = infra.jwt_issuer.JwtIssuer( "https://localhost", refresh_interval=args.jwt_key_refresh_interval_s) with jwt_issuer.start_openid_server(): txs = app.LoggingTxs(jwt_issuer=jwt_issuer) with infra.network.network( args.nodes, binary_directory=from_binary_dir, library_directory=from_library_dir, pdb=args.pdb, txs=txs, jwt_issuer=jwt_issuer, version=from_version, ) as network: network.start_and_open(args, node_container_image=from_container_image) old_nodes = network.get_joined_nodes() primary, _ = network.find_primary() LOG.info("Apply transactions to old service") issue_activity_on_live_service(network, args) new_code_id = infra.utils.get_code_id( args.enclave_type, args.oe_binary, args.package, library_dir=to_library_dir, ) network.consortium.add_new_code(primary, new_code_id) # Note: alternate between joining from snapshot and replaying entire ledger new_nodes = [] from_snapshot = True for _ in range(0, len(old_nodes)): new_node = network.create_node( "local://localhost", binary_dir=to_binary_dir, library_dir=to_library_dir, version=to_version, ) network.join_node(new_node, args.package, args, from_snapshot=from_snapshot) network.trust_node( new_node, args, valid_from=str( # Pre-2.0 nodes require X509 time format infra.crypto.datetime_to_X509time( datetime.datetime.now())), ) # For 2.x nodes joining a 1.x service before the constitution is updated, # the node certificate validity period is set by the joining node itself # as [node startup time, node startup time + 365 days] new_node.verify_certificate_validity_period( expected_validity_period_days= DEFAULT_NODE_CERTIFICATE_VALIDITY_DAYS, ignore_proposal_valid_from=True, ) from_snapshot = not from_snapshot new_nodes.append(new_node) # Verify that all nodes run the expected CCF version for node in network.get_joined_nodes(): # Note: /node/version endpoint was added in 2.x if not node.major_version or node.major_version > 1: with node.client() as c: r = c.get("/node/version") expected_version = node.version or args.ccf_version version = r.body.json()["ccf_version"] assert ( version == expected_version ), f"For node {node.local_node_id}, expect version {expected_version}, got {version}" LOG.info( "Apply transactions to hybrid network, with primary as old node" ) issue_activity_on_live_service(network, args) old_code_id = infra.utils.get_code_id( args.enclave_type, args.oe_binary, args.package, library_dir=from_library_dir, ) primary, _ = network.find_primary() network.consortium.retire_code(primary, old_code_id) for index, node in enumerate(old_nodes): network.retire_node(primary, node) if primary == node: primary, _ = network.wait_for_new_primary(primary) # This block is here to test the transition period from a network that # does not support custom claims to one that does. It can be removed after # the transition is complete. # # The new build, being unreleased, doesn't have a version at all if not primary.major_version: LOG.info("Upgrade to new JS app") # Upgrade to a version of the app containing an endpoint that # registers custom claims network.consortium.set_js_app_from_dir( primary, args.new_js_app_bundle) LOG.info("Run transaction with additional claim") # With wait_for_sync, the client checks that all nodes, including # the minority of old ones, have acked the transaction msg_idx = network.txs.idx + 1 txid = network.txs.issue(network, number_txs=1, record_claim=True, wait_for_sync=True) assert len(network.txs.pub[msg_idx]) == 1 claims = network.txs.pub[msg_idx][-1]["msg"] LOG.info( "Check receipts are fine, including transaction with claims" ) test_random_receipts( network, args, lts=True, additional_seqnos={txid.seqno: claims.encode()}, ) # Also check receipts on an old node if index + 1 < len(old_nodes): next_node = old_nodes[index + 1] test_random_receipts( network, args, lts=True, additional_seqnos={txid.seqno: None}, node=next_node, ) node.stop() LOG.info("Service is now made of new nodes only") # Rollover JWKS so that new primary must read historical CA bundle table # and retrieve new keys via auto refresh if not os.getenv("CONTAINER_NODES"): jwt_issuer.refresh_keys() # Note: /gov/jwt_keys/all endpoint was added in 2.x primary, _ = network.find_nodes() if not primary.major_version or primary.major_version > 1: jwt_issuer.wait_for_refresh(network) else: time.sleep(3) else: # https://github.com/microsoft/CCF/issues/2608#issuecomment-924785744 LOG.warning( "Skipping JWT refresh as running nodes in container") # Code update from 1.x to 2.x requires cycling the freshly-added 2.x nodes # once. This is because 2.x nodes will not have an endorsed certificate # recorded in the store and thus will not be able to have their certificate # refreshed, etc. test_new_service( network, args, to_install_path, to_binary_dir, to_library_dir, to_version, cycle_existing_nodes=True, ) # Check that the ledger can be parsed network.get_latest_ledger_public_state()
def run_code_upgrade_from( args, from_install_path, to_install_path, from_version=None, to_version=None, ): from_binary_dir, from_library_dir = get_bin_and_lib_dirs_for_install_path( from_install_path ) to_binary_dir, to_library_dir = get_bin_and_lib_dirs_for_install_path( to_install_path ) set_js_args(args, from_install_path) jwt_issuer = infra.jwt_issuer.JwtIssuer( "https://localhost", refresh_interval=args.jwt_key_refresh_interval_s ) with jwt_issuer.start_openid_server(): txs = app.LoggingTxs(jwt_issuer=jwt_issuer) with infra.network.network( args.nodes, binary_directory=from_binary_dir, library_directory=from_library_dir, pdb=args.pdb, txs=txs, jwt_issuer=jwt_issuer, version=from_version, ) as network: network.start_and_join(args) old_nodes = network.get_joined_nodes() primary, _ = network.find_primary() LOG.info("Apply transactions to old service") issue_activity_on_live_service(network, args) new_code_id = infra.utils.get_code_id( args.enclave_type, args.oe_binary, args.package, library_dir=to_library_dir, ) network.consortium.add_new_code(primary, new_code_id) # Note: alternate between joining from snapshot and replaying entire ledger new_nodes = [] from_snapshot = True for _ in range(0, len(old_nodes)): new_node = network.create_node( "local://localhost", binary_dir=to_binary_dir, library_dir=to_library_dir, version=to_version, ) network.join_node( new_node, args.package, args, from_snapshot=from_snapshot ) network.trust_node(new_node, args) # For 2.x nodes joining a 1.x service before the constitution is update, # the node certificate validity period is set by the joining node itself # as [node startup time, node startup time + 365 days] new_node.verify_certificate_validity_period( expected_validity_period_days=DEFAULT_NODE_CERTIFICATE_VALIDITY_DAYS, ignore_proposal_valid_from=True, ) from_snapshot = not from_snapshot new_nodes.append(new_node) # Verify that all nodes run the expected CCF version for node in network.get_joined_nodes(): # Note: /node/version endpoint was added in 2.x if not node.major_version or node.major_version > 1: with node.client() as c: r = c.get("/node/version") expected_version = node.version or args.ccf_version version = r.body.json()["ccf_version"] assert ( version == expected_version ), f"For node {node.local_node_id}, expect version {expected_version}, got {version}" LOG.info("Apply transactions to hybrid network, with primary as old node") issue_activity_on_live_service(network, args) old_code_id = infra.utils.get_code_id( args.enclave_type, args.oe_binary, args.package, library_dir=from_library_dir, ) primary, _ = network.find_primary() network.consortium.retire_code(primary, old_code_id) for node in old_nodes: network.retire_node(primary, node) if primary == node: primary, _ = network.wait_for_new_primary(primary) node.stop() LOG.info("Service is now made of new nodes only") # Rollover JWKS so that new primary must read historical CA bundle table # and retrieve new keys via auto refresh jwt_issuer.refresh_keys() # Note: /gov/jwt_keys/all endpoint was added in 2.x primary, _ = network.find_nodes() if not primary.major_version or primary.major_version > 1: jwt_issuer.wait_for_refresh(network) else: time.sleep(3) # Code update from 1.x to 2.x requires cycling the freshly-added 2.x nodes # once. This is because 2.x nodes will not have an endorsed certificate # recorded in the store and thus will not be able to have their certificate # refreshed, etc. test_new_service( network, args, to_install_path, to_binary_dir, to_library_dir, to_version, cycle_existing_nodes=True, ) # Check that the ledger can be parsed network.get_latest_ledger_public_state()