def test_partition_majority(network, args):
primary, backups = network.find_nodes()
# Create a partition with primary + half remaining nodes (i.e. majority)
partition = [primary]
partition.extend(backups[len(backups) // 2:])
# Wait for all nodes to be have reached the same level of commit, so that
# nodes outside of partition can become primary after this one is dropped
network.wait_for_all_nodes_to_commit(primary=primary)
# The primary should remain stable while the partition is active
# Note: Context manager
initial_view = None
with network.partitioner.partition(partition):
try:
network.wait_for_new_primary(primary)
assert False, "No new primary should be elected when partitioning majority"
except TimeoutError:
LOG.info("No new primary, as expected")
with primary.client() as c:
res = c.get("/node/network") # Well-known read-only endpoint
body = res.body.json()
initial_view = body["current_view"]
# The partitioned nodes will have called elections, increasing their view.
# When the partition is lifted, the nodes must elect a new leader, in at least this
# increased term. The winning node could come from either partition, and could even
# be the original primary.
network.wait_for_primary_unanimity(min_view=initial_view)
return network
def test_new_joiner_helps_liveness(network, args):
primary, backups = network.find_nodes()
# Issue some transactions, so there is a ledger history that a new node must receive
network.txs.issue(network, number_txs=10)
# Remove a node, leaving the network frail
network.retire_node(primary, backups[-1])
backups[-1].stop()
primary, backups = network.find_nodes()
with contextlib.ExitStack() as stack:
# Add a new node, but partition them before trusting them
new_node = network.create_node("local://localhost")
network.join_node(new_node, args.package, args, from_snapshot=False)
new_joiner_partition = [new_node]
new_joiner_rules = stack.enter_context(
network.partitioner.partition([primary, *backups],
new_joiner_partition))
# Trust the new node, and wait for commit of this (but don't ask the new node itself, which doesn't know this yet)
network.trust_node(new_node, args, no_wait=True)
check_can_progress(primary)
# Partition the primary, temporarily creating a minority service that cannot make progress
minority_partition = backups[len(backups) // 2:] + new_joiner_partition
minority_rules = stack.enter_context(
network.partitioner.partition(minority_partition))
# This is an unusual situation, where we've actually produced a dead partitioned node.
# Initially any write requests will timeout (failed attempt at forwarding), and then
# the node transitions to a candidate with nobody to talk to. Rather than trying to
# catch the errors of these states quickly, we just sleep until the latter state is
# reached, and then confirm it was reached.
time.sleep(network.observed_election_duration)
with backups[0].client("user0") as c:
r = c.post("/app/log/private", {"id": 42, "msg": "Hello world"})
assert r.status_code == http.HTTPStatus.SERVICE_UNAVAILABLE
# Restore the new node to the service
new_joiner_rules.drop()
# Confirm that the new node catches up, and progress can be made in this majority partition
network.wait_for_new_primary(primary, minority_partition)
check_can_progress(new_node)
# Explicitly drop rules before continuing
minority_rules.drop()
network.wait_for_primary_unanimity()
primary, _ = network.find_nodes()
network.wait_for_all_nodes_to_commit(primary=primary)
def test_learner_does_not_take_part(network, args):
primary, backups = network.find_nodes()
f_backups = backups[:network.get_f() + 1]
new_node = network.create_node("local://localhost")
network.join_node(new_node, args.package, args, from_snapshot=False)
with network.partitioner.partition(f_backups):
check_does_not_progress(primary, timeout=5)
try:
network.consortium.trust_node(
primary,
new_node.node_id,
timeout=ceil(args.join_timer * 2 / 1000),
valid_from=str(
infra.crypto.datetime_to_X509time(datetime.now())),
)
new_node.wait_for_node_to_join(timeout=ceil(args.join_timer * 2 /
1000))
join_failed = False
except Exception:
join_failed = True
if not join_failed:
raise Exception("join succeeded unexpectedly")
with new_node.client(self_signed_ok=True) as c:
r = c.get("/node/network/nodes/self")
assert r.body.json()["status"] == "Learner"
r = c.get("/node/consensus")
assert new_node.node_id in r.body.json()["details"]["learners"]
# New node joins, but cannot be promoted to TRUSTED without f other backups
check_does_not_progress(primary, timeout=5)
with new_node.client(self_signed_ok=True) as c:
r = c.get("/node/network/nodes/self")
assert r.body.json()["status"] == "Learner"
r = c.get("/node/consensus")
assert new_node.node_id in r.body.json()["details"]["learners"]
network.wait_for_primary_unanimity()
primary, _ = network.find_nodes()
network.wait_for_all_nodes_to_commit(primary=primary)
check_can_progress(primary)
def test_isolate_primary_from_one_backup(network, args):
p, backups = network.find_nodes()
b_0, b_1 = backups
# Issue one transaction, waiting for all nodes to be have reached
# the same level of commit, so that nodes outside of partition can
# become primary after this one is dropped
# Note: Because of https://github.com/microsoft/CCF/issues/2224, we need to
# issue a write transaction instead of just reading the TxID of the latest entry
initial_txid = network.txs.issue(network)
# Isolate first backup from primary so that first backup becomes candidate
# in a new term and wins the election
# Note: Managed manually
rules = network.partitioner.isolate_node(p, b_0)
# Now wait for several elections to occur. We expect:
# - b_0 to call and win an election with b_1's help
# - b_0 to produce a new signature, and commit it with b_1's help
# - p to call its own election, and lose because it doesn't have this signature
# - In the resulting election race:
# - If p calls first, it loses and we're in the same situation
# - If b_0 calls first, it wins, but then p calls its election and we've returned to the same situation
# - If b_1 calls first, it can win and then bring _both_ nodes up-to-date, becoming a _stable_ primary
# So we repeat elections until b_1 is primary
new_primary = network.wait_for_primary_unanimity(
min_view=initial_txid.view, timeout_multiplier=30)
assert new_primary == b_1
new_view = network.txs.issue(network).view
# The partition is now between 2 backups, but both can talk to the new primary
# Explicitly drop rules before continuing
rules.drop()
# Original primary should now, or very soon, report the new primary
new_primary_, new_view_ = network.wait_for_new_primary(p, nodes=[p])
assert (
new_primary == new_primary_
), f"New primary {new_primary_.local_node_id} after partition is dropped is different than before {new_primary.local_node_id}"
assert (
new_view == new_view_
), f"Consensus view {new_view} should not have changed after partition is dropped: now {new_view_}"
return network
def test_isolate_primary_from_one_backup(network, args):
p, backups = network.find_nodes()
b_0, b_1 = backups
# Issue one transaction, waiting for all nodes to be have reached
# the same level of commit, so that nodes outside of partition can
# become primary after this one is dropped
# Note: Because of https://github.com/microsoft/CCF/issues/2224, we need to
# issue a write transaction instead of just reading the TxID of the latest entry
initial_txid = network.txs.issue(network)
# Isolate first backup from primary so that first backup becomes candidate
# in a new term and wins the election
# Note: Managed manually
rules = network.partitioner.isolate_node(p, b_0)
LOG.info(
f"Check that primary {p.local_node_id} reports increasing last ack time for partitioned backup {b_0.local_node_id}"
)
last_ack = 0
while True:
with p.client() as c:
r = c.get("/node/consensus", log_capture=[]).body.json()["details"]
ack = r["acks"][b_0.node_id]["last_received_ms"]
if r["primary_id"] is not None:
assert (
ack >= last_ack
), f"Nodes {p.local_node_id} and {b_0.local_node_id} are no longer partitioned"
last_ack = ack
else:
LOG.debug(f"Node {p.local_node_id} is no longer primary")
break
time.sleep(0.1)
# Now wait for several elections to occur. We expect:
# - b_0 to call and win an election with b_1's help
# - b_0 to produce a new signature, and commit it with b_1's help
# - p to call its own election, and lose because it doesn't have this signature
# - In the resulting election race:
# - If p calls first, it loses and we're in the same situation
# - If b_0 calls first, it wins, but then p calls its election and we've returned to the same situation
# - If b_1 calls first, it can win and then bring _both_ nodes up-to-date, becoming a _stable_ primary
# So we repeat elections until b_1 is primary
new_primary = network.wait_for_primary_unanimity(
min_view=initial_txid.view, timeout_multiplier=30)
assert new_primary == b_1
new_view = network.txs.issue(network).view
# The partition is now between 2 backups, but both can talk to the new primary
# Explicitly drop rules before continuing
rules.drop()
LOG.info(
f"Check that new primary {new_primary.local_node_id} reports stable acks"
)
last_ack = 0
end_time = time.time() + 2 * network.args.election_timeout_ms // 1000
while time.time() < end_time:
with new_primary.client() as c:
acks = c.get("/node/consensus",
log_capture=[]).body.json()["details"]["acks"]
delayed_acks = [
ack for ack in acks.values()
if ack["last_received_ms"] > args.election_timeout_ms
]
if delayed_acks:
raise RuntimeError(
f"New primary reported some delayed acks: {acks}")
time.sleep(0.1)
# Original primary should now, or very soon, report the new primary
new_primary_, new_view_ = network.wait_for_new_primary(p, nodes=[p])
assert (
new_primary == new_primary_
), f"New primary {new_primary_.local_node_id} after partition is dropped is different than before {new_primary.local_node_id}"
assert (
new_view == new_view_
), f"Consensus view {new_view} should not have changed after partition is dropped: now {new_view_}"
return network
def test_learner_does_not_take_part(network, args):
primary, backups = network.find_nodes()
f_backups = backups[:network.get_f() + 1]
# Note: host is supplied explicitly to avoid having differently
# assigned IPs for the interfaces, something which the test infra doesn't
# support widely yet.
operator_rpc_interface = "operator_rpc_interface"
host = infra.net.expand_localhost()
new_node = network.create_node(
infra.interfaces.HostSpec(
rpc_interfaces={
infra.interfaces.PRIMARY_RPC_INTERFACE:
infra.interfaces.RPCInterface(host=host),
operator_rpc_interface:
infra.interfaces.RPCInterface(
host=host,
endorsement=infra.interfaces.Endorsement(
authority=infra.interfaces.EndorsementAuthority.Node),
),
}))
network.join_node(new_node, args.package, args, from_snapshot=False)
LOG.info("Wait for all nodes to have committed join of new pending node")
network.wait_for_all_nodes_to_commit(primary=primary)
# Here, we partition a majority of backups. This is very intentional so that
# the new learner node is not promoted to trusted while the partition is up.
# However, this means that the isolated majority of backups can (and will)
# elect one of them as new primary while the partition is up. When the partition
# is lifted, all the transactions executed of the primary node (including
# trusting the new node) will be rolled back. Because of this, we issue a new
# trust_node proposal to make sure the new node ends up being trusted and joins
# successfully.
with network.partitioner.partition(f_backups):
check_does_not_progress(primary, timeout=5)
try:
network.consortium.trust_node(
primary,
new_node.node_id,
timeout=ceil(args.join_timer_s * 2),
valid_from=datetime.now(),
)
except TimeoutError:
LOG.info("Trust node proposal did not commit as expected")
else:
raise Exception("Trust node proposal committed unexpectedly")
check_does_not_progress(primary, timeout=5)
LOG.info("Majority partition can make progress")
partition_primary, _ = network.wait_for_new_primary(primary,
nodes=f_backups)
check_can_progress(partition_primary)
LOG.info(
"New joiner is not promoted to Trusted without f other backups")
with new_node.client(interface_name=operator_rpc_interface,
verify_ca=False) as c:
r = c.get("/node/network/nodes/self")
assert r.body.json()["status"] == "Learner"
r = c.get("/node/consensus")
assert new_node.node_id in r.body.json()["details"]["learners"]
LOG.info(
"Partition is lifted, wait for primary unanimity on original nodes")
# Note: Because trusting the new node failed, the new node is not considered
# in the primary unanimity. Indeed, its transition to Trusted may have been rolled back.
primary = network.wait_for_primary_unanimity()
network.wait_for_all_nodes_to_commit(primary=primary)
LOG.info("Trust new joiner again")
network.trust_node(new_node, args)
check_can_progress(primary)
check_can_progress(new_node)