def test_ws(network, args):
primary, other = network.find_primary_and_any_backup()
msg = "Hello world"
LOG.info("Write on primary")
with primary.client("user0", ws=True) as c:
for i in [1, 50, 500]:
r = c.post("/app/log/private", {"id": 42, "msg": msg * i})
assert r.body.json() == True, r
# Before we start sending transactions to the secondary,
# we want to wait for its app frontend to be open, which is
# when it's aware that the network is open. Before that,
# we will get 404s.
end_time = time.time() + 10
with other.client("user0") as nc:
while time.time() < end_time:
r = nc.post("/app/log/private", {"id": 42, "msg": msg * i})
if r.status_code == http.HTTPStatus.OK.value:
break
else:
time.sleep(0.1)
assert r.status_code == http.HTTPStatus.OK.value, r
LOG.info("Write on secondary through forwarding")
with other.client("user0", ws=True) as c:
for i in [1, 50, 500]:
r = c.post("/app/log/private", {"id": 42, "msg": msg * i})
assert r.body.json() == True, r
return network
def test(network, args, notifications_queue=None):
primary, _ = network.find_primary_and_any_backup()
with primary.client() as mc:
check_commit = infra.checker.Checker(mc, notifications_queue)
check = infra.checker.Checker()
msg = "Hello world"
LOG.info("Write/Read on primary")
with primary.client("user0") as c:
r = c.post("/app/log/private", {"id": 42, "msg": msg})
check_commit(r, result=True)
check(c.get("/app/log/private?id=42"), result={"msg": msg})
for _ in range(10):
c.post(
"/app/log/private", {"id": 43, "msg": "Additional messages"},
)
check_commit(
c.post("/app/log/private", {"id": 43, "msg": "A final message"}),
result=True,
)
r = c.get(f"/app/receipt?commit={r.seqno}")
check(
c.post("/app/receipt/verify", {"receipt": r.body["receipt"]}),
result={"valid": True},
)
invalid = r.body["receipt"]
invalid[-3] += 1
check(
c.post("/app/receipt/verify", {"receipt": invalid}),
result={"valid": False},
)
return network
def test_node_filter(network, args):
primary, _ = network.find_primary_and_any_backup()
with primary.client() as c:
def get_nodes(status):
r = c.get(f"/node/network/nodes?status={status}")
nodes = r.body.json()["nodes"]
return sorted(nodes, key=lambda node: node["node_id"])
trusted_before = get_nodes("Trusted")
pending_before = get_nodes("Pending")
retired_before = get_nodes("Retired")
new_node = network.create_node("local://localhost")
network.join_node(new_node, args.package, args, target_node=primary)
trusted_after = get_nodes("Trusted")
pending_after = get_nodes("Pending")
retired_after = get_nodes("Retired")
assert trusted_before == trusted_after, (trusted_before, trusted_after)
assert len(pending_before) + 1 == len(pending_after), (
pending_before,
pending_after,
)
assert retired_before == retired_after, (retired_before, retired_after)
assert all(info["status"] == "Trusted" for info in trusted_after), trusted_after
assert all(info["status"] == "Pending" for info in pending_after), pending_after
assert all(info["status"] == "Retired" for info in retired_after), retired_after
return network
def test_kill_primary(network, args):
primary, backup = network.find_primary_and_any_backup()
primary.stop()
# When the consensus is BFT there is no status message timer that triggers a new election.
# It is triggered with a timeout from a message not executing. We need to send the message that
# will not execute because of the stopped primary which will then trigger a view change
if args.consensus == "bft":
try:
with backup.client("user0") as c:
_ = c.post(
"/app/log/private",
{
"id": -1,
"msg": "This is submitted to force a view change",
},
)
except CCFConnectionException:
LOG.warning(
f"Could not successfully connect to node {backup.node_id}.")
new_primary, new_term = network.wait_for_new_primary(primary.node_id)
LOG.debug(f"New primary is {new_primary.node_id} in term {new_term}")
return network
def test_receipts(network, args):
primary, _ = network.find_primary_and_any_backup()
cert_path = os.path.join(primary.common_dir,
f"{primary.local_node_id}.pem")
with open(cert_path) as c:
node_cert = load_pem_x509_certificate(c.read().encode("ascii"),
default_backend())
with primary.client() as mc:
check_commit = infra.checker.Checker(mc)
msg = "Hello world"
LOG.info("Write/Read on primary")
with primary.client("user0") as c:
for j in range(10):
idx = j + 10000
r = c.post("/app/log/private", {"id": idx, "msg": msg})
check_commit(r, result=True)
start_time = time.time()
while time.time() < (start_time + 3.0):
rc = c.get(
f"/app/receipt?transaction_id={r.view}.{r.seqno}")
if rc.status_code == http.HTTPStatus.OK:
receipt = rc.body.json()
assert receipt["root"] == ccf.receipt.root(
receipt["leaf"], receipt["proof"])
ccf.receipt.verify(receipt["root"],
receipt["signature"], node_cert)
break
elif rc.status_code == http.HTTPStatus.ACCEPTED:
time.sleep(0.5)
else:
assert False, rc
return network
def test_receipts(network, args):
primary, _ = network.find_primary_and_any_backup()
with primary.client() as mc:
check_commit = infra.checker.Checker(mc)
msg = "Hello world"
LOG.info("Write/Read on primary")
with primary.client("user0") as c:
for j in range(10):
idx = j + 10000
r = c.post("/app/log/private", {"id": idx, "msg": msg})
check_commit(r, result=True)
start_time = time.time()
while time.time() < (start_time + 3.0):
rc = c.get(f"/app/receipt?transaction_id={r.view}.{r.seqno}")
if rc.status_code == http.HTTPStatus.OK:
receipt = rc.body.json()
verify_receipt(receipt, network.cert)
break
elif rc.status_code == http.HTTPStatus.ACCEPTED:
time.sleep(0.5)
else:
assert False, rc
return network
def test_kill_primary_no_reqs(network, args):
old_primary, _ = network.find_primary_and_any_backup()
old_primary.stop()
new_primary, _ = network.wait_for_new_primary(old_primary)
# Verify that the TxID reported just after an election is valid
# Note that the first TxID read after an election may be of a signature
# Tx (time-based signature generation) in the new term rather than the
# last entry in the previous term
for node in network.get_joined_nodes():
with node.client() as c:
r = c.get("/node/network")
c.wait_for_commit(r)
# Also verify that reported last ack time are as expected
r = c.get("/node/consensus")
acks = r.body.json()["details"]["acks"]
for ack in acks.values():
if node is new_primary:
assert (ack["last_received_ms"] <
network.args.election_timeout_ms), acks
else:
assert (
ack["last_received_ms"] == 0
), f"Backup {node.local_node_id} should report time of last acks of 0: {acks}"
return network
def test_retire_primary(network, args):
pre_count = count_nodes(node_configs(network), network)
primary, backup = network.find_primary_and_any_backup()
network.consortium.retire_node(primary, primary)
network.wait_for_new_primary(primary)
check_can_progress(backup)
network.nodes.remove(primary)
post_count = count_nodes(node_configs(network), network)
assert pre_count == post_count + 1
primary.stop()
return network
def test_retire_primary(network, args):
primary, backup = network.find_primary_and_any_backup()
network.consortium.retire_node(primary, primary)
LOG.debug(
f"Waiting {network.election_duration}s for a new primary to be elected..."
)
time.sleep(network.election_duration)
new_primary, new_term = network.find_primary()
assert new_primary.node_id != primary.node_id
LOG.debug(f"New primary is {new_primary.node_id} in term {new_term}")
check_can_progress(backup)
primary.stop()
return network
def test_custom_auth(network, args):
if args.package == "liblogging":
primary, other = network.find_primary_and_any_backup()
for node in (primary, other):
with node.client() as c:
LOG.info("Request without custom headers is refused")
r = c.get("/app/custom_auth")
assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value
), r.status_code
name_header = "x-custom-auth-name"
age_header = "x-custom-auth-age"
LOG.info("Requests with partial headers are refused")
r = c.get("/app/custom_auth", headers={name_header: "Bob"})
assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value
), r.status_code
r = c.get("/app/custom_auth", headers={age_header: "42"})
assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value
), r.status_code
LOG.info(
"Requests with unacceptable header contents are refused")
r = c.get("/app/custom_auth",
headers={
name_header: "",
age_header: "42"
})
assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value
), r.status_code
r = c.get("/app/custom_auth",
headers={
name_header: "Bob",
age_header: "12"
})
assert (r.status_code == http.HTTPStatus.UNAUTHORIZED.value
), r.status_code
LOG.info("Request which meets all requirements is accepted")
r = c.get("/app/custom_auth",
headers={
name_header: "Alice",
age_header: "42"
})
assert r.status_code == http.HTTPStatus.OK.value, r.status_code
response = r.body.json()
assert response["name"] == "Alice", response
assert response["age"] == 42, response
return network
def test_random_receipts(network, args, lts=True):
primary, _ = network.find_primary_and_any_backup()
common = os.listdir(network.common_dir)
cert_paths = [
os.path.join(network.common_dir, path)
for path in common
if re.compile(r"^\d+\.pem$").match(path)
]
certs = {}
for path in cert_paths:
with open(path, encoding="utf-8") as c:
cert = c.read()
certs[infra.crypto.compute_public_key_der_hash_hex_from_pem(cert)] = cert
with primary.client("user0") as c:
r = c.get("/app/commit")
max_view, max_seqno = [
int(e) for e in r.body.json()["transaction_id"].split(".")
]
view = 2
genesis_seqno = 1
likely_first_sig_seqno = 2
last_sig_seqno = max_seqno
interesting_prefix = [genesis_seqno, likely_first_sig_seqno]
seqnos = range(len(interesting_prefix) + 1, max_seqno)
for s in (
interesting_prefix
+ sorted(random.sample(seqnos, min(50, len(seqnos))))
+ [last_sig_seqno]
):
start_time = time.time()
while time.time() < (start_time + 3.0):
rc = c.get(f"/app/receipt?transaction_id={view}.{s}")
if rc.status_code == http.HTTPStatus.OK:
receipt = rc.body.json()
if lts and not receipt.get("cert"):
receipt["cert"] = certs[receipt["node_id"]]
verify_receipt(receipt, network.cert, not lts)
if s == max_seqno:
# Always a signature receipt
assert receipt["proof"] == [], receipt
break
elif rc.status_code == http.HTTPStatus.ACCEPTED:
time.sleep(0.5)
else:
view += 1
if view > max_view:
assert False, rc
return network
def test_custom_auth_safety(network, args):
if args.package == "liblogging":
primary, other = network.find_primary_and_any_backup()
for node in (primary, other):
with node.client() as c:
r = c.get(
"/app/custom_auth",
headers={"x-custom-auth-explode": "Boom goes the dynamite"},
)
assert (
r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR.value
), r.status_code
return network
def test_retire_primary(network, args):
pre_count = count_nodes(node_configs(network), network)
primary, backup = network.find_primary_and_any_backup()
network.retire_node(primary, primary, timeout=15)
# Query this backup to find the new primary. If we ask any other
# node, then this backup may not know the new primary by the
# time we call check_can_progress.
network.wait_for_new_primary(primary, nodes=[backup])
check_can_progress(backup)
post_count = count_nodes(node_configs(network), network)
assert pre_count == post_count + 1
primary.stop()
wait_for_reconfiguration_to_complete(network)
return network
def test_kill_primary(network, args):
primary, _ = network.find_primary_and_any_backup()
primary.stop()
network.wait_for_new_primary(primary)
# Verify that the TxID reported just after an election is valid
# Note that the first TxID read after an election may be of a signature
# Tx (time-based signature generation) in the new term rather than the
# last entry in the previous term
for node in network.get_joined_nodes():
with node.client() as c:
r = c.get("/node/network")
c.wait_for_commit(r)
return network
def test_add_node_from_snapshot(
network, args, copy_ledger_read_only=True, from_backup=False
):
# Before adding the node from a snapshot, override at least one app entry
# and wait for a new committed snapshot covering that entry, so that there
# is at least one historical entry to verify.
network.txs.issue(network, number_txs=1)
for _ in range(1, args.snapshot_tx_interval):
network.txs.issue(network, number_txs=1, repeat=True)
last_tx = network.txs.get_last_tx(priv=True)
if network.wait_for_snapshot_committed_for(seqno=last_tx[1]["seqno"]):
break
target_node = None
snapshots_dir = None
if from_backup:
primary, target_node = network.find_primary_and_any_backup()
# Retrieve snapshot from primary as only primary node
# generates snapshots
snapshots_dir = network.get_committed_snapshots(primary)
new_node = network.create_node("local://localhost")
network.join_node(
new_node,
args.package,
args,
copy_ledger_read_only=copy_ledger_read_only,
target_node=target_node,
snapshots_dir=snapshots_dir,
from_snapshot=True,
)
network.trust_node(new_node, args)
with new_node.client() as c:
r = c.get("/node/state")
assert (
r.body.json()["startup_seqno"] != 0
), "Node started from snapshot but reports startup seqno of 0"
# Finally, verify all app entries on the new node, including historical ones
# from the historical ledger
network.txs.verify(node=new_node, include_historical=copy_ledger_read_only)
return network
def test_receipts(network, args):
primary, _ = network.find_primary_and_any_backup()
with primary.client() as mc:
check_commit = infra.checker.Checker(mc)
check = infra.checker.Checker()
msg = "Hello world"
LOG.info("Write/Read on primary")
with primary.client("user0") as c:
r = c.post("/app/log/private", {"id": 10000, "msg": msg})
check_commit(r, result=True)
check(c.get("/app/log/private?id=10000"), result={"msg": msg})
for _ in range(10):
c.post(
"/app/log/private",
{
"id": 10001,
"msg": "Additional messages"
},
)
check_commit(
c.post("/app/log/private", {
"id": 10001,
"msg": "A final message"
}),
result=True,
)
r = c.get(f"/app/receipt?commit={r.seqno}")
rv = c.post("/app/receipt/verify",
{"receipt": r.body.json()["receipt"]})
assert rv.body.json() == {"valid": True}
invalid = r.body.json()["receipt"]
invalid[-3] += 1
rv = c.post("/app/receipt/verify", {"receipt": invalid})
assert rv.body.json() == {"valid": False}
return network
def test_add_node_from_snapshot(
network, args, copy_ledger_read_only=True, from_backup=False
):
target_node = None
snapshot_dir = None
if from_backup:
primary, target_node = network.find_primary_and_any_backup()
# Retrieve snapshot from primary as only primary node
# generates snapshots
snapshot_dir = network.get_committed_snapshots(primary)
new_node = network.create_and_trust_node(
args.package,
"local://localhost",
args,
copy_ledger_read_only=copy_ledger_read_only,
target_node=target_node,
snapshot_dir=snapshot_dir,
)
assert new_node
return network
def test_retire_primary(network, args):
pre_count = count_nodes(node_configs(network), network)
primary, backup = network.find_primary_and_any_backup()
network.retire_node(primary, primary, timeout=15)
# Query this backup to find the new primary. If we ask any other
# node, then this backup may not know the new primary by the
# time we call check_can_progress.
new_primary, _ = network.wait_for_new_primary(primary, nodes=[backup])
# The old primary should automatically be removed from the store
# once a new primary is elected
network.wait_for_node_in_store(
new_primary,
primary.node_id,
node_status=None,
timeout=3,
)
check_can_progress(backup)
post_count = count_nodes(node_configs(network), network)
assert pre_count == post_count + 1
primary.stop()
wait_for_reconfiguration_to_complete(network)
return network
