def test_ws(network, args): primary, other = network.find_primary_and_any_backup() msg = "Hello world" LOG.info("Write on primary") with primary.client("user0", ws=True) as c: for i in [1, 50, 500]: r = c.post("/app/log/private", {"id": 42, "msg": msg * i}) assert r.body.json() == True, r # Before we start sending transactions to the secondary, # we want to wait for its app frontend to be open, which is # when it's aware that the network is open. Before that, # we will get 404s. end_time = time.time() + 10 with other.client("user0") as nc: while time.time() < end_time: r = nc.post("/app/log/private", {"id": 42, "msg": msg * i}) if r.status_code == http.HTTPStatus.OK.value: break else: time.sleep(0.1) assert r.status_code == http.HTTPStatus.OK.value, r LOG.info("Write on secondary through forwarding") with other.client("user0", ws=True) as c: for i in [1, 50, 500]: r = c.post("/app/log/private", {"id": 42, "msg": msg * i}) assert r.body.json() == True, r return network
def test(network, args, notifications_queue=None): primary, _ = network.find_primary_and_any_backup() with primary.client() as mc: check_commit = infra.checker.Checker(mc, notifications_queue) check = infra.checker.Checker() msg = "Hello world" LOG.info("Write/Read on primary") with primary.client("user0") as c: r = c.post("/app/log/private", {"id": 42, "msg": msg}) check_commit(r, result=True) check(c.get("/app/log/private?id=42"), result={"msg": msg}) for _ in range(10): c.post( "/app/log/private", {"id": 43, "msg": "Additional messages"}, ) check_commit( c.post("/app/log/private", {"id": 43, "msg": "A final message"}), result=True, ) r = c.get(f"/app/receipt?commit={r.seqno}") check( c.post("/app/receipt/verify", {"receipt": r.body["receipt"]}), result={"valid": True}, ) invalid = r.body["receipt"] invalid[-3] += 1 check( c.post("/app/receipt/verify", {"receipt": invalid}), result={"valid": False}, ) return network
def test_node_filter(network, args): primary, _ = network.find_primary_and_any_backup() with primary.client() as c: def get_nodes(status): r = c.get(f"/node/network/nodes?status={status}") nodes = r.body.json()["nodes"] return sorted(nodes, key=lambda node: node["node_id"]) trusted_before = get_nodes("Trusted") pending_before = get_nodes("Pending") retired_before = get_nodes("Retired") new_node = network.create_node("local://localhost") network.join_node(new_node, args.package, args, target_node=primary) trusted_after = get_nodes("Trusted") pending_after = get_nodes("Pending") retired_after = get_nodes("Retired") assert trusted_before == trusted_after, (trusted_before, trusted_after) assert len(pending_before) + 1 == len(pending_after), ( pending_before, pending_after, ) assert retired_before == retired_after, (retired_before, retired_after) assert all(info["status"] == "Trusted" for info in trusted_after), trusted_after assert all(info["status"] == "Pending" for info in pending_after), pending_after assert all(info["status"] == "Retired" for info in retired_after), retired_after return network
def test_kill_primary(network, args): primary, backup = network.find_primary_and_any_backup() primary.stop() # When the consensus is BFT there is no status message timer that triggers a new election. # It is triggered with a timeout from a message not executing. We need to send the message that # will not execute because of the stopped primary which will then trigger a view change if args.consensus == "bft": try: with backup.client("user0") as c: _ = c.post( "/app/log/private", { "id": -1, "msg": "This is submitted to force a view change", }, ) except CCFConnectionException: LOG.warning( f"Could not successfully connect to node {backup.node_id}.") new_primary, new_term = network.wait_for_new_primary(primary.node_id) LOG.debug(f"New primary is {new_primary.node_id} in term {new_term}") return network
def test_receipts(network, args): primary, _ = network.find_primary_and_any_backup() cert_path = os.path.join(primary.common_dir, f"{primary.local_node_id}.pem") with open(cert_path) as c: node_cert = load_pem_x509_certificate(c.read().encode("ascii"), default_backend()) with primary.client() as mc: check_commit = infra.checker.Checker(mc) msg = "Hello world" LOG.info("Write/Read on primary") with primary.client("user0") as c: for j in range(10): idx = j + 10000 r = c.post("/app/log/private", {"id": idx, "msg": msg}) check_commit(r, result=True) start_time = time.time() while time.time() < (start_time + 3.0): rc = c.get( f"/app/receipt?transaction_id={r.view}.{r.seqno}") if rc.status_code == http.HTTPStatus.OK: receipt = rc.body.json() assert receipt["root"] == ccf.receipt.root( receipt["leaf"], receipt["proof"]) ccf.receipt.verify(receipt["root"], receipt["signature"], node_cert) break elif rc.status_code == http.HTTPStatus.ACCEPTED: time.sleep(0.5) else: assert False, rc return network
def test_receipts(network, args): primary, _ = network.find_primary_and_any_backup() with primary.client() as mc: check_commit = infra.checker.Checker(mc) msg = "Hello world" LOG.info("Write/Read on primary") with primary.client("user0") as c: for j in range(10): idx = j + 10000 r = c.post("/app/log/private", {"id": idx, "msg": msg}) check_commit(r, result=True) start_time = time.time() while time.time() < (start_time + 3.0): rc = c.get(f"/app/receipt?transaction_id={r.view}.{r.seqno}") if rc.status_code == http.HTTPStatus.OK: receipt = rc.body.json() verify_receipt(receipt, network.cert) break elif rc.status_code == http.HTTPStatus.ACCEPTED: time.sleep(0.5) else: assert False, rc return network
def test_kill_primary_no_reqs(network, args): old_primary, _ = network.find_primary_and_any_backup() old_primary.stop() new_primary, _ = network.wait_for_new_primary(old_primary) # Verify that the TxID reported just after an election is valid # Note that the first TxID read after an election may be of a signature # Tx (time-based signature generation) in the new term rather than the # last entry in the previous term for node in network.get_joined_nodes(): with node.client() as c: r = c.get("/node/network") c.wait_for_commit(r) # Also verify that reported last ack time are as expected r = c.get("/node/consensus") acks = r.body.json()["details"]["acks"] for ack in acks.values(): if node is new_primary: assert (ack["last_received_ms"] < network.args.election_timeout_ms), acks else: assert ( ack["last_received_ms"] == 0 ), f"Backup {node.local_node_id} should report time of last acks of 0: {acks}" return network
def test_retire_primary(network, args): pre_count = count_nodes(node_configs(network), network) primary, backup = network.find_primary_and_any_backup() network.consortium.retire_node(primary, primary) network.wait_for_new_primary(primary) check_can_progress(backup) network.nodes.remove(primary) post_count = count_nodes(node_configs(network), network) assert pre_count == post_count + 1 primary.stop() return network
def test_retire_primary(network, args): primary, backup = network.find_primary_and_any_backup() network.consortium.retire_node(primary, primary) LOG.debug( f"Waiting {network.election_duration}s for a new primary to be elected..." ) time.sleep(network.election_duration) new_primary, new_term = network.find_primary() assert new_primary.node_id != primary.node_id LOG.debug(f"New primary is {new_primary.node_id} in term {new_term}") check_can_progress(backup) primary.stop() return network
def test_custom_auth(network, args): if args.package == "liblogging": primary, other = network.find_primary_and_any_backup() for node in (primary, other): with node.client() as c: LOG.info("Request without custom headers is refused") r = c.get("/app/custom_auth") assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value ), r.status_code name_header = "x-custom-auth-name" age_header = "x-custom-auth-age" LOG.info("Requests with partial headers are refused") r = c.get("/app/custom_auth", headers={name_header: "Bob"}) assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value ), r.status_code r = c.get("/app/custom_auth", headers={age_header: "42"}) assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value ), r.status_code LOG.info( "Requests with unacceptable header contents are refused") r = c.get("/app/custom_auth", headers={ name_header: "", age_header: "42" }) assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value ), r.status_code r = c.get("/app/custom_auth", headers={ name_header: "Bob", age_header: "12" }) assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value ), r.status_code LOG.info("Request which meets all requirements is accepted") r = c.get("/app/custom_auth", headers={ name_header: "Alice", age_header: "42" }) assert r.status_code == http.HTTPStatus.OK.value, r.status_code response = r.body.json() assert response["name"] == "Alice", response assert response["age"] == 42, response return network
def test_random_receipts(network, args, lts=True): primary, _ = network.find_primary_and_any_backup() common = os.listdir(network.common_dir) cert_paths = [ os.path.join(network.common_dir, path) for path in common if re.compile(r"^\d+\.pem$").match(path) ] certs = {} for path in cert_paths: with open(path, encoding="utf-8") as c: cert = c.read() certs[infra.crypto.compute_public_key_der_hash_hex_from_pem(cert)] = cert with primary.client("user0") as c: r = c.get("/app/commit") max_view, max_seqno = [ int(e) for e in r.body.json()["transaction_id"].split(".") ] view = 2 genesis_seqno = 1 likely_first_sig_seqno = 2 last_sig_seqno = max_seqno interesting_prefix = [genesis_seqno, likely_first_sig_seqno] seqnos = range(len(interesting_prefix) + 1, max_seqno) for s in ( interesting_prefix + sorted(random.sample(seqnos, min(50, len(seqnos)))) + [last_sig_seqno] ): start_time = time.time() while time.time() < (start_time + 3.0): rc = c.get(f"/app/receipt?transaction_id={view}.{s}") if rc.status_code == http.HTTPStatus.OK: receipt = rc.body.json() if lts and not receipt.get("cert"): receipt["cert"] = certs[receipt["node_id"]] verify_receipt(receipt, network.cert, not lts) if s == max_seqno: # Always a signature receipt assert receipt["proof"] == [], receipt break elif rc.status_code == http.HTTPStatus.ACCEPTED: time.sleep(0.5) else: view += 1 if view > max_view: assert False, rc return network
def test_custom_auth_safety(network, args): if args.package == "liblogging": primary, other = network.find_primary_and_any_backup() for node in (primary, other): with node.client() as c: r = c.get( "/app/custom_auth", headers={"x-custom-auth-explode": "Boom goes the dynamite"}, ) assert ( r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR.value ), r.status_code return network
def test_retire_primary(network, args): pre_count = count_nodes(node_configs(network), network) primary, backup = network.find_primary_and_any_backup() network.retire_node(primary, primary, timeout=15) # Query this backup to find the new primary. If we ask any other # node, then this backup may not know the new primary by the # time we call check_can_progress. network.wait_for_new_primary(primary, nodes=[backup]) check_can_progress(backup) post_count = count_nodes(node_configs(network), network) assert pre_count == post_count + 1 primary.stop() wait_for_reconfiguration_to_complete(network) return network
def test_kill_primary(network, args): primary, _ = network.find_primary_and_any_backup() primary.stop() network.wait_for_new_primary(primary) # Verify that the TxID reported just after an election is valid # Note that the first TxID read after an election may be of a signature # Tx (time-based signature generation) in the new term rather than the # last entry in the previous term for node in network.get_joined_nodes(): with node.client() as c: r = c.get("/node/network") c.wait_for_commit(r) return network
def test_add_node_from_snapshot( network, args, copy_ledger_read_only=True, from_backup=False ): # Before adding the node from a snapshot, override at least one app entry # and wait for a new committed snapshot covering that entry, so that there # is at least one historical entry to verify. network.txs.issue(network, number_txs=1) for _ in range(1, args.snapshot_tx_interval): network.txs.issue(network, number_txs=1, repeat=True) last_tx = network.txs.get_last_tx(priv=True) if network.wait_for_snapshot_committed_for(seqno=last_tx[1]["seqno"]): break target_node = None snapshots_dir = None if from_backup: primary, target_node = network.find_primary_and_any_backup() # Retrieve snapshot from primary as only primary node # generates snapshots snapshots_dir = network.get_committed_snapshots(primary) new_node = network.create_node("local://localhost") network.join_node( new_node, args.package, args, copy_ledger_read_only=copy_ledger_read_only, target_node=target_node, snapshots_dir=snapshots_dir, from_snapshot=True, ) network.trust_node(new_node, args) with new_node.client() as c: r = c.get("/node/state") assert ( r.body.json()["startup_seqno"] != 0 ), "Node started from snapshot but reports startup seqno of 0" # Finally, verify all app entries on the new node, including historical ones # from the historical ledger network.txs.verify(node=new_node, include_historical=copy_ledger_read_only) return network
def test_receipts(network, args): primary, _ = network.find_primary_and_any_backup() with primary.client() as mc: check_commit = infra.checker.Checker(mc) check = infra.checker.Checker() msg = "Hello world" LOG.info("Write/Read on primary") with primary.client("user0") as c: r = c.post("/app/log/private", {"id": 10000, "msg": msg}) check_commit(r, result=True) check(c.get("/app/log/private?id=10000"), result={"msg": msg}) for _ in range(10): c.post( "/app/log/private", { "id": 10001, "msg": "Additional messages" }, ) check_commit( c.post("/app/log/private", { "id": 10001, "msg": "A final message" }), result=True, ) r = c.get(f"/app/receipt?commit={r.seqno}") rv = c.post("/app/receipt/verify", {"receipt": r.body.json()["receipt"]}) assert rv.body.json() == {"valid": True} invalid = r.body.json()["receipt"] invalid[-3] += 1 rv = c.post("/app/receipt/verify", {"receipt": invalid}) assert rv.body.json() == {"valid": False} return network
def test_add_node_from_snapshot( network, args, copy_ledger_read_only=True, from_backup=False ): target_node = None snapshot_dir = None if from_backup: primary, target_node = network.find_primary_and_any_backup() # Retrieve snapshot from primary as only primary node # generates snapshots snapshot_dir = network.get_committed_snapshots(primary) new_node = network.create_and_trust_node( args.package, "local://localhost", args, copy_ledger_read_only=copy_ledger_read_only, target_node=target_node, snapshot_dir=snapshot_dir, ) assert new_node return network
def test_retire_primary(network, args): pre_count = count_nodes(node_configs(network), network) primary, backup = network.find_primary_and_any_backup() network.retire_node(primary, primary, timeout=15) # Query this backup to find the new primary. If we ask any other # node, then this backup may not know the new primary by the # time we call check_can_progress. new_primary, _ = network.wait_for_new_primary(primary, nodes=[backup]) # The old primary should automatically be removed from the store # once a new primary is elected network.wait_for_node_in_store( new_primary, primary.node_id, node_status=None, timeout=3, ) check_can_progress(backup) post_count = count_nodes(node_configs(network), network) assert pre_count == post_count + 1 primary.stop() wait_for_reconfiguration_to_complete(network) return network