def test_recover_service_with_expired_cert(args):
expired_service_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "expired_service"
)
new_common = infra.network.get_common_folder_name(args.workspace, args.label)
copy_tree(os.path.join(expired_service_dir, "common"), new_common)
network = infra.network.Network(args.nodes, args.binary_dir)
args.previous_service_identity_file = os.path.join(
expired_service_dir, "common", "service_cert.pem"
)
network.start_in_recovery(
args,
ledger_dir=os.path.join(expired_service_dir, "0.ledger"),
committed_ledger_dirs=[os.path.join(expired_service_dir, "0.ledger")],
snapshots_dir=os.path.join(expired_service_dir, "0.snapshots"),
common_dir=new_common,
)
network.recover(args)
primary, _ = network.find_primary()
infra.checker.check_can_progress(primary)
r = primary.get_receipt(2, 3)
verify_receipt(r.json(), network.cert)
def run(args):
hosts = args.node or ["localhost"] * 3
if not args.verbose:
LOG.remove()
LOG.add(
sys.stdout,
format="<green>[{time:YYYY-MM-DD HH:mm:ss.SSS}]</green> {message}",
)
LOG.disable("infra")
LOG.disable("ccf")
LOG.info(f"Starting {len(hosts)} CCF nodes...")
if args.enclave_type == "virtual":
LOG.warning("Virtual mode enabled")
with infra.network.network(hosts=hosts,
binary_directory=args.binary_dir,
dbg_nodes=args.debug_nodes) as network:
if args.recover:
args.label = args.label + "_recover"
LOG.info("Recovering network from:")
LOG.info(f" - Ledger: {args.ledger_dir}")
LOG.info(
f" - Defunct network public encryption key: {args.network_enc_pubk}"
)
LOG.info(f" - Common directory: {args.common_dir}")
network.start_in_recovery(args, args.ledger_dir, args.common_dir)
network.recover(args, args.network_enc_pubk)
else:
network.start_and_join(args)
primary, backups = network.find_nodes()
LOG.info("Started CCF network with the following nodes:")
LOG.info(" Node [{:2d}] = {}:{}".format(primary.node_id,
primary.pubhost,
primary.rpc_port))
for b in backups:
LOG.info(" Node [{:2d}] = {}:{}".format(b.node_id, b.pubhost,
b.rpc_port))
# Dump primary info to file for tutorial testing
if args.network_info_file is not None:
dump_network_info(args.network_info_file, network, primary)
LOG.info(
f"You can now issue business transactions to the {args.package} application."
)
LOG.info(
f"Keys and certificates have been copied to the common folder: {network.common_dir}"
)
LOG.info(
"See https://microsoft.github.io/CCF/users/issue_commands.html for more information."
)
LOG.warning("Press Ctrl+C to shutdown the network.")
try:
while True:
time.sleep(60)
except KeyboardInterrupt:
LOG.info("Stopping all CCF nodes...")
LOG.info("All CCF nodes stopped.")
def run_ledger_compatibility_since_first(args, local_branch, use_snapshot):
"""
Tests that a service from the very first LTS can be recovered
to the next LTS, and so forth, until the version of the local checkout.
The recovery process uses snapshot is `use_snapshot` is True. Otherwise, the
entire historical ledger is used.
"""
LOG.info("Use snapshot: {}", use_snapshot)
repo = infra.github.Repository()
lts_releases = repo.get_lts_releases(local_branch)
has_pre_2_rc7_ledger = False
LOG.info(f"LTS releases: {[r[1] for r in lts_releases.items()]}")
lts_versions = []
# Add an empty entry to release to indicate local checkout
# Note: dicts are ordered from Python3.7
lts_releases[None] = None
jwt_issuer = infra.jwt_issuer.JwtIssuer(
"https://localhost", refresh_interval=args.jwt_key_refresh_interval_s)
with jwt_issuer.start_openid_server():
txs = app.LoggingTxs(jwt_issuer=jwt_issuer)
for idx, (_, lts_release) in enumerate(lts_releases.items()):
if lts_release:
version, install_path = repo.install_release(lts_release)
lts_versions.append(version)
set_js_args(args, install_path)
else:
version = args.ccf_version
install_path = LOCAL_CHECKOUT_DIRECTORY
get_new_constitution_for_install(args, install_path)
binary_dir, library_dir = get_bin_and_lib_dirs_for_install_path(
install_path)
if not args.dry_run:
network_args = {
"hosts": args.nodes,
"binary_dir": binary_dir,
"library_dir": library_dir,
"txs": txs,
"jwt_issuer": jwt_issuer,
"version": version,
}
if idx == 0:
LOG.info(f"Starting new service (version: {version})")
network = infra.network.Network(**network_args)
network.start_and_open(args)
else:
LOG.info(f"Recovering service (new version: {version})")
network = infra.network.Network(**network_args,
existing_network=network)
network.start_in_recovery(
args,
ledger_dir,
committed_ledger_dirs,
snapshots_dir=snapshots_dir,
)
network.recover(args)
nodes = network.get_joined_nodes()
primary, _ = network.find_primary()
# Verify that all nodes run the expected CCF version
for node in nodes:
# Note: /node/version endpoint and custom certificate validity
# were added in 2.x
if not node.major_version or node.major_version > 1:
with node.client() as c:
r = c.get("/node/version")
expected_version = node.version or args.ccf_version
version = r.body.json()["ccf_version"]
assert (
r.body.json()["ccf_version"] ==
expected_version
), f"Node version is not {expected_version}"
node.verify_certificate_validity_period()
# Rollover JWKS so that new primary must read historical CA bundle table
# and retrieve new keys via auto refresh
jwt_issuer.refresh_keys()
# Note: /gov/jwt_keys/all endpoint was added in 2.x
primary, _ = network.find_nodes()
if not primary.major_version or primary.major_version > 1:
jwt_issuer.wait_for_refresh(network)
else:
time.sleep(3)
if idx > 0:
test_new_service(
network,
args,
install_path,
binary_dir,
library_dir,
version,
)
# We accept ledger chunk file differences during upgrades
# from 1.x to 2.x post rc7 ledger. This is necessary because
# the ledger files may not be chunked at the same interval
# between those versions (see https://github.com/microsoft/ccf/issues/3613;
# 1.x ledgers do not contain the header flags to synchronize ledger chunks).
# This can go once 2.0 is released.
current_version_past_2_rc7 = primary.version_after(
"ccf-2.0.0-rc7")
has_pre_2_rc7_ledger = (not current_version_past_2_rc7
or has_pre_2_rc7_ledger)
is_ledger_chunk_breaking = (has_pre_2_rc7_ledger
and current_version_past_2_rc7)
snapshots_dir = (network.get_committed_snapshots(primary)
if use_snapshot else None)
network.stop_all_nodes(
skip_verification=True,
accept_ledger_diff=is_ledger_chunk_breaking,
)
ledger_dir, committed_ledger_dirs = primary.get_ledger()
network.save_service_identity(args)
# Check that ledger and snapshots can be parsed
ccf.ledger.Ledger(
committed_ledger_dirs).get_latest_public_state()
if snapshots_dir:
for s in os.listdir(snapshots_dir):
with ccf.ledger.Snapshot(os.path.join(
snapshots_dir, s)) as snapshot:
snapshot.get_public_domain()
return lts_versions
def run(args):
hosts = args.node or DEFAULT_NODES
if not args.verbose:
LOG.remove()
LOG.add(
sys.stdout,
format="<green>[{time:HH:mm:ss.SSS}]</green> {message}",
)
LOG.disable("infra")
LOG.disable("ccf")
LOG.info(
f"Starting {len(hosts)} CCF node{'s' if len(hosts) > 1 else ''}...")
if args.enclave_type == "virtual":
LOG.warning("Virtual mode enabled")
with infra.network.network(
hosts=hosts,
binary_directory=args.binary_dir,
library_directory=args.library_dir,
dbg_nodes=args.debug_nodes,
) as network:
if args.recover:
args.label = args.label + "_recover"
LOG.info("Recovering network from:")
LOG.info(f" - Common directory: {args.common_dir}")
LOG.info(f" - Ledger: {args.ledger_dir}")
if args.snapshot_dir:
LOG.info(f" - Snapshots: {args.snapshot_dir}")
else:
LOG.warning(
"No available snapshot to recover from. Entire transaction history will be replayed."
)
network.start_in_recovery(
args,
args.ledger_dir,
snapshot_dir=args.snapshot_dir,
common_dir=args.common_dir,
)
network.recover(args)
else:
network.start_and_join(args)
primary, backups = network.find_nodes()
max_len = len(str(len(backups)))
# To be sure, confirm that the app frontend is open on each node
for node in [primary, *backups]:
with node.client("user0") as c:
if args.verbose:
r = c.get("/app/commit")
else:
r = c.get("/app/commit", log_capture=[])
assert r.status_code == http.HTTPStatus.OK, r.status_code
def pad_node_id(nid):
return (f"{{:{max_len}d}}").format(nid)
LOG.info("Started CCF network with the following nodes:")
LOG.info(" Node [{}] = https://{}:{}".format(
pad_node_id(primary.node_id), primary.pubhost, primary.rpc_port))
for b in backups:
LOG.info(" Node [{}] = https://{}:{}".format(
pad_node_id(b.node_id), b.pubhost, b.rpc_port))
LOG.info(
f"You can now issue business transactions to the {args.package} application."
)
LOG.info(
f"Keys and certificates have been copied to the common folder: {network.common_dir}"
)
LOG.info(
"See https://microsoft.github.io/CCF/master/users/issue_commands.html for more information."
)
LOG.warning("Press Ctrl+C to shutdown the network.")
try:
while True:
time.sleep(60)
except KeyboardInterrupt:
LOG.info("Stopping all CCF nodes...")
LOG.info("All CCF nodes stopped.")
def run(args):
# Read RPC interfaces from configuration file if specified, otherwise
# read from command line arguments or default
if args.config_file is not None:
with open(args.config_file, encoding="utf-8") as f:
hosts = [
infra.interfaces.HostSpec.from_json(
json.load(f)["network"]["rpc_interfaces"])
]
else:
hosts = args.node or DEFAULT_NODES
hosts = [infra.interfaces.HostSpec.from_str(node) for node in hosts]
if not args.verbose:
LOG.remove()
LOG.add(
sys.stdout,
format="<green>[{time:HH:mm:ss.SSS}]</green> {message}",
)
LOG.disable("infra")
LOG.disable("ccf")
if args.enclave_type == "virtual":
LOG.warning("Virtual mode enabled")
LOG.info(
f"Starting {len(hosts)} CCF node{'s' if len(hosts) > 1 else ''}...")
try:
with infra.network.network(
hosts=hosts,
binary_directory=args.binary_dir,
library_directory=args.library_dir,
dbg_nodes=args.debug_nodes,
) as network:
if args.recover:
args.label = args.label + "_recover"
LOG.info("Recovering network from:")
LOG.info(f" - Common directory: {args.common_dir}")
LOG.info(f" - Ledger: {args.ledger_dir}")
if args.snapshots_dir:
LOG.info(f" - Snapshots: {args.snapshots_dir}")
else:
LOG.warning(
"No available snapshot to recover from. Entire transaction history will be replayed."
)
args.previous_service_identity_file = os.path.join(
args.common_dir, "service_cert.pem")
network.start_in_recovery(
args,
args.ledger_dir,
snapshots_dir=args.snapshots_dir,
common_dir=args.common_dir,
)
network.recover(args)
else:
network.start_and_open(args)
nodes = network.get_joined_nodes()
max_len = max([len(str(node.local_node_id)) for node in nodes])
# To be sure, confirm that the app frontend is open on each node
for node in nodes:
with node.client() as c:
if args.verbose:
r = c.get("/app/commit")
else:
r = c.get("/app/commit", log_capture=[])
assert r.status_code == http.HTTPStatus.OK, r.status_code
def pad_node_id(nid):
return (f"{{:{max_len}d}}").format(nid)
LOG.info("Started CCF network with the following nodes:")
for node in nodes:
LOG.info(" Node [{}] = https://{}:{}".format(
pad_node_id(node.local_node_id),
node.get_public_rpc_host(),
node.get_public_rpc_port(),
))
LOG.info(
f"You can now issue business transactions to the {args.package} application"
)
if args.js_app_bundle is not None:
LOG.info(f"Loaded JS application: {args.js_app_bundle}")
LOG.info(
f"Keys and certificates have been copied to the common folder: {network.common_dir}"
)
LOG.info(
"See https://microsoft.github.io/CCF/main/use_apps/issue_commands.html for more information"
)
LOG.warning("Press Ctrl+C to shutdown the network")
try:
while True:
time.sleep(60)
except KeyboardInterrupt:
LOG.info("Stopping all CCF nodes...")
LOG.info("All CCF nodes stopped.")
except infra.network.NetworkShutdownError as e:
LOG.error("Error! Some nodes ran into issues:")
for node_id, errors in e.errors.items():
errors = "\n".join(errors)
LOG.error(f"- Node [{node_id}]:\n{errors}\n")
LOG.info(
"Please raise a bug if the issue is unexpected: https://github.com/microsoft/CCF/issues/new?assignees=&labels=bug&template=bug_report.md&title=Unexpected%20error%20when%20running%20sandbox%20script"
)
sys.exit(1)
def run_ledger_compatibility_since_first(args, local_branch, use_snapshot):
"""
Tests that a service from the very first LTS can be recovered
to the next LTS, and so forth, until the version of the local checkout.
The recovery process uses snapshot is `use_snapshot` is True. Otherwise, the
entire historical ledger is used.
"""
LOG.info("Use snapshot: {}", use_snapshot)
repo = infra.github.Repository()
lts_releases = repo.get_lts_releases()
LOG.info(f"LTS releases: {[r[1] for r in lts_releases.items()]}")
lts_versions = []
# Add an empty entry to release to indicate local checkout
# Note: dicts are ordered from Python3.7
lts_releases[None] = None
jwt_issuer = infra.jwt_issuer.JwtIssuer(
"https://localhost", refresh_interval=args.jwt_key_refresh_interval_s
)
with jwt_issuer.start_openid_server():
txs = app.LoggingTxs(jwt_issuer=jwt_issuer)
for idx, (_, lts_release) in enumerate(lts_releases.items()):
if lts_release:
version, install_path = repo.install_release(lts_release)
lts_versions.append(version)
set_js_args(args, install_path)
else:
version = args.ccf_version
install_path = LOCAL_CHECKOUT_DIRECTORY
binary_dir, library_dir = get_bin_and_lib_dirs_for_install_path(
install_path
)
if not args.dry_run:
network_args = {
"hosts": args.nodes,
"binary_dir": binary_dir,
"library_dir": library_dir,
"txs": txs,
"jwt_issuer": jwt_issuer,
"version": version,
}
if idx == 0:
LOG.info(f"Starting new service (version: {version})")
network = infra.network.Network(**network_args)
network.start_and_join(args)
else:
LOG.info(f"Recovering service (new version: {version})")
network = infra.network.Network(
**network_args, existing_network=network
)
network.start_in_recovery(
args,
ledger_dir,
committed_ledger_dir,
snapshot_dir=snapshot_dir,
)
network.recover(args)
nodes = network.get_joined_nodes()
primary, _ = network.find_primary()
# Verify that all nodes run the expected CCF version
for node in nodes:
# Note: /node/version endpoint and custom certificate validity
# were added in 2.x
if not node.major_version or node.major_version > 1:
with node.client() as c:
r = c.get("/node/version")
expected_version = node.version or args.ccf_version
version = r.body.json()["ccf_version"]
assert (
r.body.json()["ccf_version"] == expected_version
), f"Node version is not {expected_version}"
node.verify_certificate_validity_period()
# Rollover JWKS so that new primary must read historical CA bundle table
# and retrieve new keys via auto refresh
jwt_issuer.refresh_keys()
# Note: /gov/jwt_keys/all endpoint was added in 2.x
primary, _ = network.find_nodes()
if not primary.major_version or primary.major_version > 1:
jwt_issuer.wait_for_refresh(network)
else:
time.sleep(3)
if idx > 0:
test_new_service(
network,
args,
install_path,
binary_dir,
library_dir,
version,
)
snapshot_dir = (
network.get_committed_snapshots(primary) if use_snapshot else None
)
ledger_dir, committed_ledger_dir = primary.get_ledger(
include_read_only_dirs=True
)
network.stop_all_nodes(skip_verification=True)
# Check that ledger and snapshots can be parsed
ccf.ledger.Ledger([committed_ledger_dir]).get_latest_public_state()
if snapshot_dir:
for s in os.listdir(snapshot_dir):
with ccf.ledger.Snapshot(
os.path.join(snapshot_dir, s)
) as snapshot:
snapshot.get_public_domain()
return lts_versions
