def test_recover_service_with_expired_cert(args): expired_service_dir = os.path.join( os.path.dirname(os.path.realpath(__file__)), "expired_service" ) new_common = infra.network.get_common_folder_name(args.workspace, args.label) copy_tree(os.path.join(expired_service_dir, "common"), new_common) network = infra.network.Network(args.nodes, args.binary_dir) args.previous_service_identity_file = os.path.join( expired_service_dir, "common", "service_cert.pem" ) network.start_in_recovery( args, ledger_dir=os.path.join(expired_service_dir, "0.ledger"), committed_ledger_dirs=[os.path.join(expired_service_dir, "0.ledger")], snapshots_dir=os.path.join(expired_service_dir, "0.snapshots"), common_dir=new_common, ) network.recover(args) primary, _ = network.find_primary() infra.checker.check_can_progress(primary) r = primary.get_receipt(2, 3) verify_receipt(r.json(), network.cert)
def run(args): hosts = args.node or ["localhost"] * 3 if not args.verbose: LOG.remove() LOG.add( sys.stdout, format="<green>[{time:YYYY-MM-DD HH:mm:ss.SSS}]</green> {message}", ) LOG.disable("infra") LOG.disable("ccf") LOG.info(f"Starting {len(hosts)} CCF nodes...") if args.enclave_type == "virtual": LOG.warning("Virtual mode enabled") with infra.network.network(hosts=hosts, binary_directory=args.binary_dir, dbg_nodes=args.debug_nodes) as network: if args.recover: args.label = args.label + "_recover" LOG.info("Recovering network from:") LOG.info(f" - Ledger: {args.ledger_dir}") LOG.info( f" - Defunct network public encryption key: {args.network_enc_pubk}" ) LOG.info(f" - Common directory: {args.common_dir}") network.start_in_recovery(args, args.ledger_dir, args.common_dir) network.recover(args, args.network_enc_pubk) else: network.start_and_join(args) primary, backups = network.find_nodes() LOG.info("Started CCF network with the following nodes:") LOG.info(" Node [{:2d}] = {}:{}".format(primary.node_id, primary.pubhost, primary.rpc_port)) for b in backups: LOG.info(" Node [{:2d}] = {}:{}".format(b.node_id, b.pubhost, b.rpc_port)) # Dump primary info to file for tutorial testing if args.network_info_file is not None: dump_network_info(args.network_info_file, network, primary) LOG.info( f"You can now issue business transactions to the {args.package} application." ) LOG.info( f"Keys and certificates have been copied to the common folder: {network.common_dir}" ) LOG.info( "See https://microsoft.github.io/CCF/users/issue_commands.html for more information." ) LOG.warning("Press Ctrl+C to shutdown the network.") try: while True: time.sleep(60) except KeyboardInterrupt: LOG.info("Stopping all CCF nodes...") LOG.info("All CCF nodes stopped.")
def run_ledger_compatibility_since_first(args, local_branch, use_snapshot): """ Tests that a service from the very first LTS can be recovered to the next LTS, and so forth, until the version of the local checkout. The recovery process uses snapshot is `use_snapshot` is True. Otherwise, the entire historical ledger is used. """ LOG.info("Use snapshot: {}", use_snapshot) repo = infra.github.Repository() lts_releases = repo.get_lts_releases(local_branch) has_pre_2_rc7_ledger = False LOG.info(f"LTS releases: {[r[1] for r in lts_releases.items()]}") lts_versions = [] # Add an empty entry to release to indicate local checkout # Note: dicts are ordered from Python3.7 lts_releases[None] = None jwt_issuer = infra.jwt_issuer.JwtIssuer( "https://localhost", refresh_interval=args.jwt_key_refresh_interval_s) with jwt_issuer.start_openid_server(): txs = app.LoggingTxs(jwt_issuer=jwt_issuer) for idx, (_, lts_release) in enumerate(lts_releases.items()): if lts_release: version, install_path = repo.install_release(lts_release) lts_versions.append(version) set_js_args(args, install_path) else: version = args.ccf_version install_path = LOCAL_CHECKOUT_DIRECTORY get_new_constitution_for_install(args, install_path) binary_dir, library_dir = get_bin_and_lib_dirs_for_install_path( install_path) if not args.dry_run: network_args = { "hosts": args.nodes, "binary_dir": binary_dir, "library_dir": library_dir, "txs": txs, "jwt_issuer": jwt_issuer, "version": version, } if idx == 0: LOG.info(f"Starting new service (version: {version})") network = infra.network.Network(**network_args) network.start_and_open(args) else: LOG.info(f"Recovering service (new version: {version})") network = infra.network.Network(**network_args, existing_network=network) network.start_in_recovery( args, ledger_dir, committed_ledger_dirs, snapshots_dir=snapshots_dir, ) network.recover(args) nodes = network.get_joined_nodes() primary, _ = network.find_primary() # Verify that all nodes run the expected CCF version for node in nodes: # Note: /node/version endpoint and custom certificate validity # were added in 2.x if not node.major_version or node.major_version > 1: with node.client() as c: r = c.get("/node/version") expected_version = node.version or args.ccf_version version = r.body.json()["ccf_version"] assert ( r.body.json()["ccf_version"] == expected_version ), f"Node version is not {expected_version}" node.verify_certificate_validity_period() # Rollover JWKS so that new primary must read historical CA bundle table # and retrieve new keys via auto refresh jwt_issuer.refresh_keys() # Note: /gov/jwt_keys/all endpoint was added in 2.x primary, _ = network.find_nodes() if not primary.major_version or primary.major_version > 1: jwt_issuer.wait_for_refresh(network) else: time.sleep(3) if idx > 0: test_new_service( network, args, install_path, binary_dir, library_dir, version, ) # We accept ledger chunk file differences during upgrades # from 1.x to 2.x post rc7 ledger. This is necessary because # the ledger files may not be chunked at the same interval # between those versions (see https://github.com/microsoft/ccf/issues/3613; # 1.x ledgers do not contain the header flags to synchronize ledger chunks). # This can go once 2.0 is released. current_version_past_2_rc7 = primary.version_after( "ccf-2.0.0-rc7") has_pre_2_rc7_ledger = (not current_version_past_2_rc7 or has_pre_2_rc7_ledger) is_ledger_chunk_breaking = (has_pre_2_rc7_ledger and current_version_past_2_rc7) snapshots_dir = (network.get_committed_snapshots(primary) if use_snapshot else None) network.stop_all_nodes( skip_verification=True, accept_ledger_diff=is_ledger_chunk_breaking, ) ledger_dir, committed_ledger_dirs = primary.get_ledger() network.save_service_identity(args) # Check that ledger and snapshots can be parsed ccf.ledger.Ledger( committed_ledger_dirs).get_latest_public_state() if snapshots_dir: for s in os.listdir(snapshots_dir): with ccf.ledger.Snapshot(os.path.join( snapshots_dir, s)) as snapshot: snapshot.get_public_domain() return lts_versions
def run(args): hosts = args.node or DEFAULT_NODES if not args.verbose: LOG.remove() LOG.add( sys.stdout, format="<green>[{time:HH:mm:ss.SSS}]</green> {message}", ) LOG.disable("infra") LOG.disable("ccf") LOG.info( f"Starting {len(hosts)} CCF node{'s' if len(hosts) > 1 else ''}...") if args.enclave_type == "virtual": LOG.warning("Virtual mode enabled") with infra.network.network( hosts=hosts, binary_directory=args.binary_dir, library_directory=args.library_dir, dbg_nodes=args.debug_nodes, ) as network: if args.recover: args.label = args.label + "_recover" LOG.info("Recovering network from:") LOG.info(f" - Common directory: {args.common_dir}") LOG.info(f" - Ledger: {args.ledger_dir}") if args.snapshot_dir: LOG.info(f" - Snapshots: {args.snapshot_dir}") else: LOG.warning( "No available snapshot to recover from. Entire transaction history will be replayed." ) network.start_in_recovery( args, args.ledger_dir, snapshot_dir=args.snapshot_dir, common_dir=args.common_dir, ) network.recover(args) else: network.start_and_join(args) primary, backups = network.find_nodes() max_len = len(str(len(backups))) # To be sure, confirm that the app frontend is open on each node for node in [primary, *backups]: with node.client("user0") as c: if args.verbose: r = c.get("/app/commit") else: r = c.get("/app/commit", log_capture=[]) assert r.status_code == http.HTTPStatus.OK, r.status_code def pad_node_id(nid): return (f"{{:{max_len}d}}").format(nid) LOG.info("Started CCF network with the following nodes:") LOG.info(" Node [{}] = https://{}:{}".format( pad_node_id(primary.node_id), primary.pubhost, primary.rpc_port)) for b in backups: LOG.info(" Node [{}] = https://{}:{}".format( pad_node_id(b.node_id), b.pubhost, b.rpc_port)) LOG.info( f"You can now issue business transactions to the {args.package} application." ) LOG.info( f"Keys and certificates have been copied to the common folder: {network.common_dir}" ) LOG.info( "See https://microsoft.github.io/CCF/master/users/issue_commands.html for more information." ) LOG.warning("Press Ctrl+C to shutdown the network.") try: while True: time.sleep(60) except KeyboardInterrupt: LOG.info("Stopping all CCF nodes...") LOG.info("All CCF nodes stopped.")
def run(args): # Read RPC interfaces from configuration file if specified, otherwise # read from command line arguments or default if args.config_file is not None: with open(args.config_file, encoding="utf-8") as f: hosts = [ infra.interfaces.HostSpec.from_json( json.load(f)["network"]["rpc_interfaces"]) ] else: hosts = args.node or DEFAULT_NODES hosts = [infra.interfaces.HostSpec.from_str(node) for node in hosts] if not args.verbose: LOG.remove() LOG.add( sys.stdout, format="<green>[{time:HH:mm:ss.SSS}]</green> {message}", ) LOG.disable("infra") LOG.disable("ccf") if args.enclave_type == "virtual": LOG.warning("Virtual mode enabled") LOG.info( f"Starting {len(hosts)} CCF node{'s' if len(hosts) > 1 else ''}...") try: with infra.network.network( hosts=hosts, binary_directory=args.binary_dir, library_directory=args.library_dir, dbg_nodes=args.debug_nodes, ) as network: if args.recover: args.label = args.label + "_recover" LOG.info("Recovering network from:") LOG.info(f" - Common directory: {args.common_dir}") LOG.info(f" - Ledger: {args.ledger_dir}") if args.snapshots_dir: LOG.info(f" - Snapshots: {args.snapshots_dir}") else: LOG.warning( "No available snapshot to recover from. Entire transaction history will be replayed." ) args.previous_service_identity_file = os.path.join( args.common_dir, "service_cert.pem") network.start_in_recovery( args, args.ledger_dir, snapshots_dir=args.snapshots_dir, common_dir=args.common_dir, ) network.recover(args) else: network.start_and_open(args) nodes = network.get_joined_nodes() max_len = max([len(str(node.local_node_id)) for node in nodes]) # To be sure, confirm that the app frontend is open on each node for node in nodes: with node.client() as c: if args.verbose: r = c.get("/app/commit") else: r = c.get("/app/commit", log_capture=[]) assert r.status_code == http.HTTPStatus.OK, r.status_code def pad_node_id(nid): return (f"{{:{max_len}d}}").format(nid) LOG.info("Started CCF network with the following nodes:") for node in nodes: LOG.info(" Node [{}] = https://{}:{}".format( pad_node_id(node.local_node_id), node.get_public_rpc_host(), node.get_public_rpc_port(), )) LOG.info( f"You can now issue business transactions to the {args.package} application" ) if args.js_app_bundle is not None: LOG.info(f"Loaded JS application: {args.js_app_bundle}") LOG.info( f"Keys and certificates have been copied to the common folder: {network.common_dir}" ) LOG.info( "See https://microsoft.github.io/CCF/main/use_apps/issue_commands.html for more information" ) LOG.warning("Press Ctrl+C to shutdown the network") try: while True: time.sleep(60) except KeyboardInterrupt: LOG.info("Stopping all CCF nodes...") LOG.info("All CCF nodes stopped.") except infra.network.NetworkShutdownError as e: LOG.error("Error! Some nodes ran into issues:") for node_id, errors in e.errors.items(): errors = "\n".join(errors) LOG.error(f"- Node [{node_id}]:\n{errors}\n") LOG.info( "Please raise a bug if the issue is unexpected: https://github.com/microsoft/CCF/issues/new?assignees=&labels=bug&template=bug_report.md&title=Unexpected%20error%20when%20running%20sandbox%20script" ) sys.exit(1)
def run_ledger_compatibility_since_first(args, local_branch, use_snapshot): """ Tests that a service from the very first LTS can be recovered to the next LTS, and so forth, until the version of the local checkout. The recovery process uses snapshot is `use_snapshot` is True. Otherwise, the entire historical ledger is used. """ LOG.info("Use snapshot: {}", use_snapshot) repo = infra.github.Repository() lts_releases = repo.get_lts_releases() LOG.info(f"LTS releases: {[r[1] for r in lts_releases.items()]}") lts_versions = [] # Add an empty entry to release to indicate local checkout # Note: dicts are ordered from Python3.7 lts_releases[None] = None jwt_issuer = infra.jwt_issuer.JwtIssuer( "https://localhost", refresh_interval=args.jwt_key_refresh_interval_s ) with jwt_issuer.start_openid_server(): txs = app.LoggingTxs(jwt_issuer=jwt_issuer) for idx, (_, lts_release) in enumerate(lts_releases.items()): if lts_release: version, install_path = repo.install_release(lts_release) lts_versions.append(version) set_js_args(args, install_path) else: version = args.ccf_version install_path = LOCAL_CHECKOUT_DIRECTORY binary_dir, library_dir = get_bin_and_lib_dirs_for_install_path( install_path ) if not args.dry_run: network_args = { "hosts": args.nodes, "binary_dir": binary_dir, "library_dir": library_dir, "txs": txs, "jwt_issuer": jwt_issuer, "version": version, } if idx == 0: LOG.info(f"Starting new service (version: {version})") network = infra.network.Network(**network_args) network.start_and_join(args) else: LOG.info(f"Recovering service (new version: {version})") network = infra.network.Network( **network_args, existing_network=network ) network.start_in_recovery( args, ledger_dir, committed_ledger_dir, snapshot_dir=snapshot_dir, ) network.recover(args) nodes = network.get_joined_nodes() primary, _ = network.find_primary() # Verify that all nodes run the expected CCF version for node in nodes: # Note: /node/version endpoint and custom certificate validity # were added in 2.x if not node.major_version or node.major_version > 1: with node.client() as c: r = c.get("/node/version") expected_version = node.version or args.ccf_version version = r.body.json()["ccf_version"] assert ( r.body.json()["ccf_version"] == expected_version ), f"Node version is not {expected_version}" node.verify_certificate_validity_period() # Rollover JWKS so that new primary must read historical CA bundle table # and retrieve new keys via auto refresh jwt_issuer.refresh_keys() # Note: /gov/jwt_keys/all endpoint was added in 2.x primary, _ = network.find_nodes() if not primary.major_version or primary.major_version > 1: jwt_issuer.wait_for_refresh(network) else: time.sleep(3) if idx > 0: test_new_service( network, args, install_path, binary_dir, library_dir, version, ) snapshot_dir = ( network.get_committed_snapshots(primary) if use_snapshot else None ) ledger_dir, committed_ledger_dir = primary.get_ledger( include_read_only_dirs=True ) network.stop_all_nodes(skip_verification=True) # Check that ledger and snapshots can be parsed ccf.ledger.Ledger([committed_ledger_dir]).get_latest_public_state() if snapshot_dir: for s in os.listdir(snapshot_dir): with ccf.ledger.Snapshot( os.path.join(snapshot_dir, s) ) as snapshot: snapshot.get_public_domain() return lts_versions