def run(args):
# Keep track of governance operations that happened in the test
governance_operations = set()
with infra.network.network(args.nodes,
args.binary_dir,
args.debug_nodes,
args.perf_nodes,
pdb=args.pdb) as network:
network.start_and_join(args)
primary, _ = network.find_primary()
ledger_directories = primary.remote.ledger_paths()
LOG.info("Add new member proposal (implicit vote)")
(
new_member_proposal,
_,
careful_vote,
) = network.consortium.generate_and_propose_new_member(
primary, curve=infra.network.EllipticCurve.secp256r1)
member = network.consortium.get_member_by_local_id(
new_member_proposal.proposer_id)
governance_operations.add(
(new_member_proposal.proposal_id, member.service_id, "propose"))
LOG.info("2/3 members accept the proposal")
p = network.consortium.vote_using_majority(primary,
new_member_proposal,
careful_vote)
for voter in p.voters:
governance_operations.add((p.proposal_id, voter, "vote"))
assert new_member_proposal.state == infra.proposal.ProposalState.ACCEPTED
LOG.info("Create new proposal but withdraw it before it is accepted")
new_member_proposal, _, _ = network.consortium.generate_and_propose_new_member(
primary, curve=infra.network.EllipticCurve.secp256r1)
member = network.consortium.get_member_by_local_id(
new_member_proposal.proposer_id)
governance_operations.add(
(new_member_proposal.proposal_id, member.service_id, "propose"))
with primary.client() as c:
response = network.consortium.get_member_by_local_id(
new_member_proposal.proposer_id).withdraw(
primary, new_member_proposal)
infra.checker.Checker(c)(response)
assert response.status_code == http.HTTPStatus.OK.value
assert response.body.json()["state"] == ProposalState.WITHDRAWN.value
member = network.consortium.get_member_by_local_id(
new_member_proposal.proposer_id)
governance_operations.add(
(new_member_proposal.proposal_id, member.service_id, "withdraw"))
# Force ledger flush of all transactions so far
network.get_latest_ledger_public_state()
ledger = ccf.ledger.Ledger(ledger_directories)
check_operations(ledger, governance_operations)
test_ledger_is_readable(network, args)
test_tables_doc(network, args)
def test_service_principals(network, args):
node = network.find_node_by_role()
principal_id = "0xdeadbeef"
# Initially, there is nothing in this table
latest_public_tables, _ = network.get_latest_ledger_public_state()
assert "public:ccf.gov.service_principals" not in latest_public_tables
# Create and accept a proposal which populates an entry in this table
principal_data = {"name": "Bob", "roles": ["Fireman", "Zookeeper"]}
proposal = {
"actions": [{
"name": "set_service_principal",
"args": {
"id": principal_id,
"data": principal_data
},
}]
}
ballot = {
"ballot":
"export function vote(proposal, proposer_id) { return true; }"
}
proposal = network.consortium.get_any_active_member().propose(
node, proposal)
network.consortium.vote_using_majority(node, proposal, ballot)
# Confirm it can be read
latest_public_tables, _ = network.get_latest_ledger_public_state()
assert (json.loads(
latest_public_tables["public:ccf.gov.service_principals"][
principal_id.encode()]) == principal_data)
# Create and accept a proposal which removes an entry from this table
proposal = {
"actions": [{
"name": "remove_service_principal",
"args": {
"id": principal_id
}
}]
}
proposal = network.consortium.get_any_active_member().propose(
node, proposal)
network.consortium.vote_using_majority(node, proposal, ballot)
# Confirm it is gone
latest_public_tables, _ = network.get_latest_ledger_public_state()
assert (principal_id.encode()
not in latest_public_tables["public:ccf.gov.service_principals"])
return network
def get_current_nodes_table(network):
tables, _ = network.get_latest_ledger_public_state()
tn = "public:ccf.gov.nodes.info"
r = {}
for nid, info in tables[tn].items():
r[nid.decode()] = json.loads(info)
return r
def test_read_ledger_utility(network, args):
def fmt_str(data: bytes) -> str:
return data.decode()
format_rule = [(".*records.*", {"key": fmt_str, "value": fmt_str})]
# Issue at least one transaction to see how it is read in the ledger
network.txs.issue(network, number_txs=1)
network.get_latest_ledger_public_state()
primary, backups = network.find_nodes()
for node in (primary, *backups):
ledger_dirs = node.remote.ledger_paths()
assert ccf.read_ledger.run(ledger_dirs,
tables_format_rules=format_rule)
return network
def test_retiring_nodes_emit_at_most_one_signature(network, args):
primary, _ = network.find_primary()
# Force ledger flush of all transactions so far
network.get_latest_ledger_public_state()
ledger = ccf.ledger.Ledger(primary.remote.ledger_paths())
retiring_nodes = set()
retired_nodes = set()
for chunk in ledger:
for tr in chunk:
tables = tr.get_public_domain().get_tables()
if ccf.ledger.NODES_TABLE_NAME in tables:
nodes = tables[ccf.ledger.NODES_TABLE_NAME]
for nid, info_ in nodes.items():
if info_ is None:
# Node was removed
continue
info = json.loads(info_)
if info["status"] == "Retired":
retiring_nodes.add(nid)
if ccf.ledger.SIGNATURE_TX_TABLE_NAME in tables:
sigs = tables[ccf.ledger.SIGNATURE_TX_TABLE_NAME]
assert len(sigs) == 1, sigs.keys()
(sig_,) = sigs.values()
sig = json.loads(sig_)
assert (
sig["node"] not in retired_nodes
), f"Unexpected signature from {sig['node']}"
retired_nodes |= retiring_nodes
retiring_nodes = set()
assert not retiring_nodes, (retiring_nodes, retired_nodes)
LOG.info("{} nodes retired throughout test", len(retired_nodes))
wait_for_reconfiguration_to_complete(network)
return network
def check_kv_jwt_key_matches(network, kid, cert_pem):
latest_public_state, _ = network.get_latest_ledger_public_state()
latest_jwt_signing_key = latest_public_state[
"public:ccf.gov.jwt.public_signing_keys"
]
if cert_pem is None:
assert kid.encode() not in latest_jwt_signing_key
else:
stored_cert = infra.crypto.cert_der_to_pem(latest_jwt_signing_key[kid.encode()])
assert infra.crypto.are_certs_equal(
cert_pem, stored_cert
), "input cert is not equal to stored cert"
def test_read_ledger_utility(network, args):
def fmt_str(data: bytes) -> str:
return data.decode()
format_rule = [(".*records.*", {"key": fmt_str, "value": fmt_str})]
network.txs.issue(network, number_txs=args.snapshot_tx_interval)
network.get_latest_ledger_public_state()
primary, backups = network.find_nodes()
for node in (primary, *backups):
ledger_dirs = node.remote.ledger_paths()
assert ccf.read_ledger.run(paths=ledger_dirs,
tables_format_rules=format_rule)
snapshot_dir = network.get_committed_snapshots(primary)
assert ccf.read_ledger.run(
paths=[os.path.join(snapshot_dir,
os.listdir(snapshot_dir)[-1])],
is_snapshot=True,
tables_format_rules=format_rule,
)
return network
def test_member_data(network, args):
assert args.initial_operator_count > 0
latest_public_tables, _ = network.get_latest_ledger_public_state()
members_info = latest_public_tables["public:ccf.gov.members.info"]
md_count = 0
for member in network.get_members():
stored_member_info = json.loads(members_info[member.service_id.encode()])
if member.member_data:
assert (
stored_member_info["member_data"] == member.member_data
), f'stored member data "{stored_member_info["member_data"]}" != expected "{member.member_data} "'
md_count += 1
else:
assert "member_data" not in stored_member_info
assert md_count == args.initial_operator_count
return network
def run_code_upgrade_from(
args,
from_install_path,
to_install_path,
from_version=None,
to_version=None,
from_container_image=None,
):
from_binary_dir, from_library_dir = get_bin_and_lib_dirs_for_install_path(
from_install_path)
to_binary_dir, to_library_dir = get_bin_and_lib_dirs_for_install_path(
to_install_path)
set_js_args(args, from_install_path, to_install_path)
jwt_issuer = infra.jwt_issuer.JwtIssuer(
"https://localhost", refresh_interval=args.jwt_key_refresh_interval_s)
with jwt_issuer.start_openid_server():
txs = app.LoggingTxs(jwt_issuer=jwt_issuer)
with infra.network.network(
args.nodes,
binary_directory=from_binary_dir,
library_directory=from_library_dir,
pdb=args.pdb,
txs=txs,
jwt_issuer=jwt_issuer,
version=from_version,
) as network:
network.start_and_open(args,
node_container_image=from_container_image)
old_nodes = network.get_joined_nodes()
primary, _ = network.find_primary()
LOG.info("Apply transactions to old service")
issue_activity_on_live_service(network, args)
new_code_id = infra.utils.get_code_id(
args.enclave_type,
args.oe_binary,
args.package,
library_dir=to_library_dir,
)
network.consortium.add_new_code(primary, new_code_id)
# Note: alternate between joining from snapshot and replaying entire ledger
new_nodes = []
from_snapshot = True
for _ in range(0, len(old_nodes)):
new_node = network.create_node(
"local://localhost",
binary_dir=to_binary_dir,
library_dir=to_library_dir,
version=to_version,
)
network.join_node(new_node,
args.package,
args,
from_snapshot=from_snapshot)
network.trust_node(
new_node,
args,
valid_from=str( # Pre-2.0 nodes require X509 time format
infra.crypto.datetime_to_X509time(
datetime.datetime.now())),
)
# For 2.x nodes joining a 1.x service before the constitution is updated,
# the node certificate validity period is set by the joining node itself
# as [node startup time, node startup time + 365 days]
new_node.verify_certificate_validity_period(
expected_validity_period_days=
DEFAULT_NODE_CERTIFICATE_VALIDITY_DAYS,
ignore_proposal_valid_from=True,
)
from_snapshot = not from_snapshot
new_nodes.append(new_node)
# Verify that all nodes run the expected CCF version
for node in network.get_joined_nodes():
# Note: /node/version endpoint was added in 2.x
if not node.major_version or node.major_version > 1:
with node.client() as c:
r = c.get("/node/version")
expected_version = node.version or args.ccf_version
version = r.body.json()["ccf_version"]
assert (
version == expected_version
), f"For node {node.local_node_id}, expect version {expected_version}, got {version}"
LOG.info(
"Apply transactions to hybrid network, with primary as old node"
)
issue_activity_on_live_service(network, args)
old_code_id = infra.utils.get_code_id(
args.enclave_type,
args.oe_binary,
args.package,
library_dir=from_library_dir,
)
primary, _ = network.find_primary()
network.consortium.retire_code(primary, old_code_id)
for index, node in enumerate(old_nodes):
network.retire_node(primary, node)
if primary == node:
primary, _ = network.wait_for_new_primary(primary)
# This block is here to test the transition period from a network that
# does not support custom claims to one that does. It can be removed after
# the transition is complete.
#
# The new build, being unreleased, doesn't have a version at all
if not primary.major_version:
LOG.info("Upgrade to new JS app")
# Upgrade to a version of the app containing an endpoint that
# registers custom claims
network.consortium.set_js_app_from_dir(
primary, args.new_js_app_bundle)
LOG.info("Run transaction with additional claim")
# With wait_for_sync, the client checks that all nodes, including
# the minority of old ones, have acked the transaction
msg_idx = network.txs.idx + 1
txid = network.txs.issue(network,
number_txs=1,
record_claim=True,
wait_for_sync=True)
assert len(network.txs.pub[msg_idx]) == 1
claims = network.txs.pub[msg_idx][-1]["msg"]
LOG.info(
"Check receipts are fine, including transaction with claims"
)
test_random_receipts(
network,
args,
lts=True,
additional_seqnos={txid.seqno: claims.encode()},
)
# Also check receipts on an old node
if index + 1 < len(old_nodes):
next_node = old_nodes[index + 1]
test_random_receipts(
network,
args,
lts=True,
additional_seqnos={txid.seqno: None},
node=next_node,
)
node.stop()
LOG.info("Service is now made of new nodes only")
# Rollover JWKS so that new primary must read historical CA bundle table
# and retrieve new keys via auto refresh
if not os.getenv("CONTAINER_NODES"):
jwt_issuer.refresh_keys()
# Note: /gov/jwt_keys/all endpoint was added in 2.x
primary, _ = network.find_nodes()
if not primary.major_version or primary.major_version > 1:
jwt_issuer.wait_for_refresh(network)
else:
time.sleep(3)
else:
# https://github.com/microsoft/CCF/issues/2608#issuecomment-924785744
LOG.warning(
"Skipping JWT refresh as running nodes in container")
# Code update from 1.x to 2.x requires cycling the freshly-added 2.x nodes
# once. This is because 2.x nodes will not have an endorsed certificate
# recorded in the store and thus will not be able to have their certificate
# refreshed, etc.
test_new_service(
network,
args,
to_install_path,
to_binary_dir,
to_library_dir,
to_version,
cycle_existing_nodes=True,
)
# Check that the ledger can be parsed
network.get_latest_ledger_public_state()
def run_code_upgrade_from(
args,
from_install_path,
to_install_path,
from_version=None,
to_version=None,
):
from_binary_dir, from_library_dir = get_bin_and_lib_dirs_for_install_path(
from_install_path
)
to_binary_dir, to_library_dir = get_bin_and_lib_dirs_for_install_path(
to_install_path
)
set_js_args(args, from_install_path)
jwt_issuer = infra.jwt_issuer.JwtIssuer(
"https://localhost", refresh_interval=args.jwt_key_refresh_interval_s
)
with jwt_issuer.start_openid_server():
txs = app.LoggingTxs(jwt_issuer=jwt_issuer)
with infra.network.network(
args.nodes,
binary_directory=from_binary_dir,
library_directory=from_library_dir,
pdb=args.pdb,
txs=txs,
jwt_issuer=jwt_issuer,
version=from_version,
) as network:
network.start_and_join(args)
old_nodes = network.get_joined_nodes()
primary, _ = network.find_primary()
LOG.info("Apply transactions to old service")
issue_activity_on_live_service(network, args)
new_code_id = infra.utils.get_code_id(
args.enclave_type,
args.oe_binary,
args.package,
library_dir=to_library_dir,
)
network.consortium.add_new_code(primary, new_code_id)
# Note: alternate between joining from snapshot and replaying entire ledger
new_nodes = []
from_snapshot = True
for _ in range(0, len(old_nodes)):
new_node = network.create_node(
"local://localhost",
binary_dir=to_binary_dir,
library_dir=to_library_dir,
version=to_version,
)
network.join_node(
new_node, args.package, args, from_snapshot=from_snapshot
)
network.trust_node(new_node, args)
# For 2.x nodes joining a 1.x service before the constitution is update,
# the node certificate validity period is set by the joining node itself
# as [node startup time, node startup time + 365 days]
new_node.verify_certificate_validity_period(
expected_validity_period_days=DEFAULT_NODE_CERTIFICATE_VALIDITY_DAYS,
ignore_proposal_valid_from=True,
)
from_snapshot = not from_snapshot
new_nodes.append(new_node)
# Verify that all nodes run the expected CCF version
for node in network.get_joined_nodes():
# Note: /node/version endpoint was added in 2.x
if not node.major_version or node.major_version > 1:
with node.client() as c:
r = c.get("/node/version")
expected_version = node.version or args.ccf_version
version = r.body.json()["ccf_version"]
assert (
version == expected_version
), f"For node {node.local_node_id}, expect version {expected_version}, got {version}"
LOG.info("Apply transactions to hybrid network, with primary as old node")
issue_activity_on_live_service(network, args)
old_code_id = infra.utils.get_code_id(
args.enclave_type,
args.oe_binary,
args.package,
library_dir=from_library_dir,
)
primary, _ = network.find_primary()
network.consortium.retire_code(primary, old_code_id)
for node in old_nodes:
network.retire_node(primary, node)
if primary == node:
primary, _ = network.wait_for_new_primary(primary)
node.stop()
LOG.info("Service is now made of new nodes only")
# Rollover JWKS so that new primary must read historical CA bundle table
# and retrieve new keys via auto refresh
jwt_issuer.refresh_keys()
# Note: /gov/jwt_keys/all endpoint was added in 2.x
primary, _ = network.find_nodes()
if not primary.major_version or primary.major_version > 1:
jwt_issuer.wait_for_refresh(network)
else:
time.sleep(3)
# Code update from 1.x to 2.x requires cycling the freshly-added 2.x nodes
# once. This is because 2.x nodes will not have an endorsed certificate
# recorded in the store and thus will not be able to have their certificate
# refreshed, etc.
test_new_service(
network,
args,
to_install_path,
to_binary_dir,
to_library_dir,
to_version,
cycle_existing_nodes=True,
)
# Check that the ledger can be parsed
network.get_latest_ledger_public_state()