def run_ledger_compatibility_since_first(args, local_branch, use_snapshot):
"""
Tests that a service from the very first LTS can be recovered
to the next LTS, and so forth, until the version of the local checkout.
The recovery process uses snapshot is `use_snapshot` is True. Otherwise, the
entire historical ledger is used.
"""
LOG.info("Use snapshot: {}", use_snapshot)
repo = infra.github.Repository()
lts_releases = repo.get_lts_releases(local_branch)
has_pre_2_rc7_ledger = False
LOG.info(f"LTS releases: {[r[1] for r in lts_releases.items()]}")
lts_versions = []
# Add an empty entry to release to indicate local checkout
# Note: dicts are ordered from Python3.7
lts_releases[None] = None
jwt_issuer = infra.jwt_issuer.JwtIssuer(
"https://localhost", refresh_interval=args.jwt_key_refresh_interval_s)
with jwt_issuer.start_openid_server():
txs = app.LoggingTxs(jwt_issuer=jwt_issuer)
for idx, (_, lts_release) in enumerate(lts_releases.items()):
if lts_release:
version, install_path = repo.install_release(lts_release)
lts_versions.append(version)
set_js_args(args, install_path)
else:
version = args.ccf_version
install_path = LOCAL_CHECKOUT_DIRECTORY
get_new_constitution_for_install(args, install_path)
binary_dir, library_dir = get_bin_and_lib_dirs_for_install_path(
install_path)
if not args.dry_run:
network_args = {
"hosts": args.nodes,
"binary_dir": binary_dir,
"library_dir": library_dir,
"txs": txs,
"jwt_issuer": jwt_issuer,
"version": version,
}
if idx == 0:
LOG.info(f"Starting new service (version: {version})")
network = infra.network.Network(**network_args)
network.start_and_open(args)
else:
LOG.info(f"Recovering service (new version: {version})")
network = infra.network.Network(**network_args,
existing_network=network)
network.start_in_recovery(
args,
ledger_dir,
committed_ledger_dirs,
snapshots_dir=snapshots_dir,
)
network.recover(args)
nodes = network.get_joined_nodes()
primary, _ = network.find_primary()
# Verify that all nodes run the expected CCF version
for node in nodes:
# Note: /node/version endpoint and custom certificate validity
# were added in 2.x
if not node.major_version or node.major_version > 1:
with node.client() as c:
r = c.get("/node/version")
expected_version = node.version or args.ccf_version
version = r.body.json()["ccf_version"]
assert (
r.body.json()["ccf_version"] ==
expected_version
), f"Node version is not {expected_version}"
node.verify_certificate_validity_period()
# Rollover JWKS so that new primary must read historical CA bundle table
# and retrieve new keys via auto refresh
jwt_issuer.refresh_keys()
# Note: /gov/jwt_keys/all endpoint was added in 2.x
primary, _ = network.find_nodes()
if not primary.major_version or primary.major_version > 1:
jwt_issuer.wait_for_refresh(network)
else:
time.sleep(3)
if idx > 0:
test_new_service(
network,
args,
install_path,
binary_dir,
library_dir,
version,
)
# We accept ledger chunk file differences during upgrades
# from 1.x to 2.x post rc7 ledger. This is necessary because
# the ledger files may not be chunked at the same interval
# between those versions (see https://github.com/microsoft/ccf/issues/3613;
# 1.x ledgers do not contain the header flags to synchronize ledger chunks).
# This can go once 2.0 is released.
current_version_past_2_rc7 = primary.version_after(
"ccf-2.0.0-rc7")
has_pre_2_rc7_ledger = (not current_version_past_2_rc7
or has_pre_2_rc7_ledger)
is_ledger_chunk_breaking = (has_pre_2_rc7_ledger
and current_version_past_2_rc7)
snapshots_dir = (network.get_committed_snapshots(primary)
if use_snapshot else None)
network.stop_all_nodes(
skip_verification=True,
accept_ledger_diff=is_ledger_chunk_breaking,
)
ledger_dir, committed_ledger_dirs = primary.get_ledger()
network.save_service_identity(args)
# Check that ledger and snapshots can be parsed
ccf.ledger.Ledger(
committed_ledger_dirs).get_latest_public_state()
if snapshots_dir:
for s in os.listdir(snapshots_dir):
with ccf.ledger.Snapshot(os.path.join(
snapshots_dir, s)) as snapshot:
snapshot.get_public_domain()
return lts_versions
def run_ledger_compatibility_since_first(args, local_branch, use_snapshot):
"""
Tests that a service from the very first LTS can be recovered
to the next LTS, and so forth, until the version of the local checkout.
The recovery process uses snapshot is `use_snapshot` is True. Otherwise, the
entire historical ledger is used.
"""
LOG.info("Use snapshot: {}", use_snapshot)
repo = infra.github.Repository()
lts_releases = repo.get_lts_releases()
LOG.info(f"LTS releases: {[r[1] for r in lts_releases.items()]}")
lts_versions = []
# Add an empty entry to release to indicate local checkout
# Note: dicts are ordered from Python3.7
lts_releases[None] = None
jwt_issuer = infra.jwt_issuer.JwtIssuer(
"https://localhost", refresh_interval=args.jwt_key_refresh_interval_s
)
with jwt_issuer.start_openid_server():
txs = app.LoggingTxs(jwt_issuer=jwt_issuer)
for idx, (_, lts_release) in enumerate(lts_releases.items()):
if lts_release:
version, install_path = repo.install_release(lts_release)
lts_versions.append(version)
set_js_args(args, install_path)
else:
version = args.ccf_version
install_path = LOCAL_CHECKOUT_DIRECTORY
binary_dir, library_dir = get_bin_and_lib_dirs_for_install_path(
install_path
)
if not args.dry_run:
network_args = {
"hosts": args.nodes,
"binary_dir": binary_dir,
"library_dir": library_dir,
"txs": txs,
"jwt_issuer": jwt_issuer,
"version": version,
}
if idx == 0:
LOG.info(f"Starting new service (version: {version})")
network = infra.network.Network(**network_args)
network.start_and_join(args)
else:
LOG.info(f"Recovering service (new version: {version})")
network = infra.network.Network(
**network_args, existing_network=network
)
network.start_in_recovery(
args,
ledger_dir,
committed_ledger_dir,
snapshot_dir=snapshot_dir,
)
network.recover(args)
nodes = network.get_joined_nodes()
primary, _ = network.find_primary()
# Verify that all nodes run the expected CCF version
for node in nodes:
# Note: /node/version endpoint and custom certificate validity
# were added in 2.x
if not node.major_version or node.major_version > 1:
with node.client() as c:
r = c.get("/node/version")
expected_version = node.version or args.ccf_version
version = r.body.json()["ccf_version"]
assert (
r.body.json()["ccf_version"] == expected_version
), f"Node version is not {expected_version}"
node.verify_certificate_validity_period()
# Rollover JWKS so that new primary must read historical CA bundle table
# and retrieve new keys via auto refresh
jwt_issuer.refresh_keys()
# Note: /gov/jwt_keys/all endpoint was added in 2.x
primary, _ = network.find_nodes()
if not primary.major_version or primary.major_version > 1:
jwt_issuer.wait_for_refresh(network)
else:
time.sleep(3)
if idx > 0:
test_new_service(
network,
args,
install_path,
binary_dir,
library_dir,
version,
)
snapshot_dir = (
network.get_committed_snapshots(primary) if use_snapshot else None
)
ledger_dir, committed_ledger_dir = primary.get_ledger(
include_read_only_dirs=True
)
network.stop_all_nodes(skip_verification=True)
# Check that ledger and snapshots can be parsed
ccf.ledger.Ledger([committed_ledger_dir]).get_latest_public_state()
if snapshot_dir:
for s in os.listdir(snapshot_dir):
with ccf.ledger.Snapshot(
os.path.join(snapshot_dir, s)
) as snapshot:
snapshot.get_public_domain()
return lts_versions
def run_code_upgrade_from(
args,
from_install_path,
to_install_path,
from_version=None,
to_version=None,
from_container_image=None,
):
from_binary_dir, from_library_dir = get_bin_and_lib_dirs_for_install_path(
from_install_path)
to_binary_dir, to_library_dir = get_bin_and_lib_dirs_for_install_path(
to_install_path)
set_js_args(args, from_install_path, to_install_path)
jwt_issuer = infra.jwt_issuer.JwtIssuer(
"https://localhost", refresh_interval=args.jwt_key_refresh_interval_s)
with jwt_issuer.start_openid_server():
txs = app.LoggingTxs(jwt_issuer=jwt_issuer)
with infra.network.network(
args.nodes,
binary_directory=from_binary_dir,
library_directory=from_library_dir,
pdb=args.pdb,
txs=txs,
jwt_issuer=jwt_issuer,
version=from_version,
) as network:
network.start_and_open(args,
node_container_image=from_container_image)
old_nodes = network.get_joined_nodes()
primary, _ = network.find_primary()
LOG.info("Apply transactions to old service")
issue_activity_on_live_service(network, args)
new_code_id = infra.utils.get_code_id(
args.enclave_type,
args.oe_binary,
args.package,
library_dir=to_library_dir,
)
network.consortium.add_new_code(primary, new_code_id)
# Note: alternate between joining from snapshot and replaying entire ledger
new_nodes = []
from_snapshot = True
for _ in range(0, len(old_nodes)):
new_node = network.create_node(
"local://localhost",
binary_dir=to_binary_dir,
library_dir=to_library_dir,
version=to_version,
)
network.join_node(new_node,
args.package,
args,
from_snapshot=from_snapshot)
network.trust_node(
new_node,
args,
valid_from=str( # Pre-2.0 nodes require X509 time format
infra.crypto.datetime_to_X509time(
datetime.datetime.now())),
)
# For 2.x nodes joining a 1.x service before the constitution is updated,
# the node certificate validity period is set by the joining node itself
# as [node startup time, node startup time + 365 days]
new_node.verify_certificate_validity_period(
expected_validity_period_days=
DEFAULT_NODE_CERTIFICATE_VALIDITY_DAYS,
ignore_proposal_valid_from=True,
)
from_snapshot = not from_snapshot
new_nodes.append(new_node)
# Verify that all nodes run the expected CCF version
for node in network.get_joined_nodes():
# Note: /node/version endpoint was added in 2.x
if not node.major_version or node.major_version > 1:
with node.client() as c:
r = c.get("/node/version")
expected_version = node.version or args.ccf_version
version = r.body.json()["ccf_version"]
assert (
version == expected_version
), f"For node {node.local_node_id}, expect version {expected_version}, got {version}"
LOG.info(
"Apply transactions to hybrid network, with primary as old node"
)
issue_activity_on_live_service(network, args)
old_code_id = infra.utils.get_code_id(
args.enclave_type,
args.oe_binary,
args.package,
library_dir=from_library_dir,
)
primary, _ = network.find_primary()
network.consortium.retire_code(primary, old_code_id)
for index, node in enumerate(old_nodes):
network.retire_node(primary, node)
if primary == node:
primary, _ = network.wait_for_new_primary(primary)
# This block is here to test the transition period from a network that
# does not support custom claims to one that does. It can be removed after
# the transition is complete.
#
# The new build, being unreleased, doesn't have a version at all
if not primary.major_version:
LOG.info("Upgrade to new JS app")
# Upgrade to a version of the app containing an endpoint that
# registers custom claims
network.consortium.set_js_app_from_dir(
primary, args.new_js_app_bundle)
LOG.info("Run transaction with additional claim")
# With wait_for_sync, the client checks that all nodes, including
# the minority of old ones, have acked the transaction
msg_idx = network.txs.idx + 1
txid = network.txs.issue(network,
number_txs=1,
record_claim=True,
wait_for_sync=True)
assert len(network.txs.pub[msg_idx]) == 1
claims = network.txs.pub[msg_idx][-1]["msg"]
LOG.info(
"Check receipts are fine, including transaction with claims"
)
test_random_receipts(
network,
args,
lts=True,
additional_seqnos={txid.seqno: claims.encode()},
)
# Also check receipts on an old node
if index + 1 < len(old_nodes):
next_node = old_nodes[index + 1]
test_random_receipts(
network,
args,
lts=True,
additional_seqnos={txid.seqno: None},
node=next_node,
)
node.stop()
LOG.info("Service is now made of new nodes only")
# Rollover JWKS so that new primary must read historical CA bundle table
# and retrieve new keys via auto refresh
if not os.getenv("CONTAINER_NODES"):
jwt_issuer.refresh_keys()
# Note: /gov/jwt_keys/all endpoint was added in 2.x
primary, _ = network.find_nodes()
if not primary.major_version or primary.major_version > 1:
jwt_issuer.wait_for_refresh(network)
else:
time.sleep(3)
else:
# https://github.com/microsoft/CCF/issues/2608#issuecomment-924785744
LOG.warning(
"Skipping JWT refresh as running nodes in container")
# Code update from 1.x to 2.x requires cycling the freshly-added 2.x nodes
# once. This is because 2.x nodes will not have an endorsed certificate
# recorded in the store and thus will not be able to have their certificate
# refreshed, etc.
test_new_service(
network,
args,
to_install_path,
to_binary_dir,
to_library_dir,
to_version,
cycle_existing_nodes=True,
)
# Check that the ledger can be parsed
network.get_latest_ledger_public_state()
def run_code_upgrade_from(
args,
from_install_path,
to_install_path,
from_version=None,
to_version=None,
):
from_binary_dir, from_library_dir = get_bin_and_lib_dirs_for_install_path(
from_install_path
)
to_binary_dir, to_library_dir = get_bin_and_lib_dirs_for_install_path(
to_install_path
)
set_js_args(args, from_install_path)
jwt_issuer = infra.jwt_issuer.JwtIssuer(
"https://localhost", refresh_interval=args.jwt_key_refresh_interval_s
)
with jwt_issuer.start_openid_server():
txs = app.LoggingTxs(jwt_issuer=jwt_issuer)
with infra.network.network(
args.nodes,
binary_directory=from_binary_dir,
library_directory=from_library_dir,
pdb=args.pdb,
txs=txs,
jwt_issuer=jwt_issuer,
version=from_version,
) as network:
network.start_and_join(args)
old_nodes = network.get_joined_nodes()
primary, _ = network.find_primary()
LOG.info("Apply transactions to old service")
issue_activity_on_live_service(network, args)
new_code_id = infra.utils.get_code_id(
args.enclave_type,
args.oe_binary,
args.package,
library_dir=to_library_dir,
)
network.consortium.add_new_code(primary, new_code_id)
# Note: alternate between joining from snapshot and replaying entire ledger
new_nodes = []
from_snapshot = True
for _ in range(0, len(old_nodes)):
new_node = network.create_node(
"local://localhost",
binary_dir=to_binary_dir,
library_dir=to_library_dir,
version=to_version,
)
network.join_node(
new_node, args.package, args, from_snapshot=from_snapshot
)
network.trust_node(new_node, args)
# For 2.x nodes joining a 1.x service before the constitution is update,
# the node certificate validity period is set by the joining node itself
# as [node startup time, node startup time + 365 days]
new_node.verify_certificate_validity_period(
expected_validity_period_days=DEFAULT_NODE_CERTIFICATE_VALIDITY_DAYS,
ignore_proposal_valid_from=True,
)
from_snapshot = not from_snapshot
new_nodes.append(new_node)
# Verify that all nodes run the expected CCF version
for node in network.get_joined_nodes():
# Note: /node/version endpoint was added in 2.x
if not node.major_version or node.major_version > 1:
with node.client() as c:
r = c.get("/node/version")
expected_version = node.version or args.ccf_version
version = r.body.json()["ccf_version"]
assert (
version == expected_version
), f"For node {node.local_node_id}, expect version {expected_version}, got {version}"
LOG.info("Apply transactions to hybrid network, with primary as old node")
issue_activity_on_live_service(network, args)
old_code_id = infra.utils.get_code_id(
args.enclave_type,
args.oe_binary,
args.package,
library_dir=from_library_dir,
)
primary, _ = network.find_primary()
network.consortium.retire_code(primary, old_code_id)
for node in old_nodes:
network.retire_node(primary, node)
if primary == node:
primary, _ = network.wait_for_new_primary(primary)
node.stop()
LOG.info("Service is now made of new nodes only")
# Rollover JWKS so that new primary must read historical CA bundle table
# and retrieve new keys via auto refresh
jwt_issuer.refresh_keys()
# Note: /gov/jwt_keys/all endpoint was added in 2.x
primary, _ = network.find_nodes()
if not primary.major_version or primary.major_version > 1:
jwt_issuer.wait_for_refresh(network)
else:
time.sleep(3)
# Code update from 1.x to 2.x requires cycling the freshly-added 2.x nodes
# once. This is because 2.x nodes will not have an endorsed certificate
# recorded in the store and thus will not be able to have their certificate
# refreshed, etc.
test_new_service(
network,
args,
to_install_path,
to_binary_dir,
to_library_dir,
to_version,
cycle_existing_nodes=True,
)
# Check that the ledger can be parsed
network.get_latest_ledger_public_state()
