def test_retire_backup(network, args):
primary, _ = network.find_primary()
backup_to_retire = network.find_any_backup()
network.consortium.retire_node(primary, backup_to_retire)
backup_to_retire.stop()
check_can_progress(primary)
return network
def test_add_node_from_backup(network, args):
new_node = network.create_node("local://localhost")
network.join_node(
new_node, args.package, args, target_node=network.find_any_backup()
)
network.trust_node(new_node, args)
return network
def test_forwarding_frontends(network, args):
backup = network.find_any_backup()
with backup.client() as c:
check_commit = infra.checker.Checker(c)
ack = network.consortium.get_any_active_member().ack(backup)
check_commit(ack)
with backup.client("user0") as c:
check_commit = infra.checker.Checker(c)
check = infra.checker.Checker()
msg = "forwarded_msg"
log_id = 123
check_commit(
c.post("/app/log/private", {
"id": log_id,
"msg": msg
}),
result=True,
)
check(c.get(f"/app/log/private?id={log_id}"), result={"msg": msg})
if args.package == "liblogging":
escaped_query_tests(c, "request_query")
return network
def test_add_node_from_backup(network, args):
backup = network.find_any_backup()
new_node = network.create_and_trust_node(
args.package, "localhost", args, target_node=backup
)
assert new_node
return network
def test_retire_backup(network, args):
primary, _ = network.find_primary()
backup_to_retire = network.find_any_backup()
network.retire_node(primary, backup_to_retire)
backup_to_retire.stop()
check_can_progress(primary)
wait_for_reconfiguration_to_complete(network)
return network
def test_primary(network, args):
primary, _ = network.find_primary()
with primary.client() as c:
r = c.head("/node/primary")
assert r.status_code == http.HTTPStatus.OK.value
backup = network.find_any_backup()
with backup.client() as c:
r = c.head("/node/primary", allow_redirects=False)
assert r.status_code == http.HTTPStatus.PERMANENT_REDIRECT.value
assert (r.headers["location"] ==
f"https://{primary.pubhost}:{primary.pubport}/node/primary")
return network
def test_retire_backup(network, args):
primary, _ = network.find_primary()
backup_to_retire = network.find_any_backup()
network.retire_node(primary, backup_to_retire)
network.wait_for_node_in_store(
primary,
backup_to_retire.node_id,
node_status=None,
timeout=3,
)
backup_to_retire.stop()
check_can_progress(primary)
wait_for_reconfiguration_to_complete(network)
return network
def test_primary(network, args, notifications_queue=None, verify=True):
LOG.error(network.nodes)
primary, _ = network.find_primary()
LOG.error(f"PRIMARY {primary.pubhost}")
with primary.client() as c:
r = c.head("/node/primary")
assert r.status_code == http.HTTPStatus.OK.value
backup = network.find_any_backup()
LOG.error(f"BACKUP {backup.pubhost}")
with backup.client() as c:
r = c.head("/node/primary")
assert r.status_code == http.HTTPStatus.PERMANENT_REDIRECT.value
assert (r.headers["location"] ==
f"https://{primary.pubhost}:{primary.rpc_port}/node/primary")
return network
def test_primary(network, args):
primary, _ = network.find_primary()
with primary.client() as c:
r = c.head("/node/primary")
assert r.status_code == http.HTTPStatus.OK.value
backup = network.find_any_backup()
for interface_name in backup.host.rpc_interfaces.keys():
with backup.client(interface_name=interface_name) as c:
r = c.head("/node/primary", allow_redirects=False)
assert r.status_code == http.HTTPStatus.PERMANENT_REDIRECT.value
primary_interface = primary.host.rpc_interfaces[interface_name]
assert (
r.headers["location"] ==
f"https://{primary_interface.public_host}:{primary_interface.public_port}/node/primary"
)
LOG.info(
f'Successfully redirected to {r.headers["location"]} on primary {primary.local_node_id}'
)
return network
def run(args):
hosts = ["localhost", "localhost", "localhost"]
LOG.info(f"setting seed to {args.seed}")
random.seed(args.seed)
txs = app.LoggingTxs()
with infra.network.network(
hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs
) as network:
network.start_and_join(args)
original_nodes = network.get_joined_nodes()
view_info = {}
suspend.update_view_info(network, view_info)
app.test_run_txs(network=network, args=args, num_txs=TOTAL_REQUESTS)
suspend.update_view_info(network, view_info)
nodes_to_kill = [network.find_any_backup()]
nodes_to_keep = [n for n in original_nodes if n not in nodes_to_kill]
# check that a new node can catch up after all the requests
late_joiner = network.create_and_trust_node(args.package, "localhost", args)
nodes_to_keep.append(late_joiner)
# some requests to be processed while the late joiner catches up
# (no strict checking that these requests are actually being processed simultaneously with the node catchup)
app.test_run_txs(
network=network,
args=args,
num_txs=int(TOTAL_REQUESTS / 2),
nodes=original_nodes, # doesn't contain late joiner
verify=False, # will try to verify for late joiner and it might not be ready yet
)
suspend.wait_for_late_joiner(original_nodes[0], late_joiner)
# kill the old node(s) and ensure we are still making progress
for backup_to_retire in nodes_to_kill:
LOG.success(f"Stopping node {backup_to_retire.node_id}")
backup_to_retire.stop()
# check nodes are ok after we killed one off
app.test_run_txs(
network=network,
args=args,
nodes=nodes_to_keep,
num_txs=len(nodes_to_keep),
timeout=30,
ignore_failures=True,
# in the event of an early view change due to the late joiner this might
# take longer than usual to complete and we don't want the test to break here
)
suspend.test_suspend_nodes(network, args, nodes_to_keep)
# run txs while nodes get suspended
app.test_run_txs(
network=network,
args=args,
num_txs=4 * TOTAL_REQUESTS,
timeout=30,
ignore_failures=True,
# in the event of an early view change due to the late joiner this might
# take longer than usual to complete and we don't want the test to break here
)
suspend.update_view_info(network, view_info)
# check nodes have resumed normal execution before shutting down
app.test_run_txs(network=network, args=args, num_txs=len(nodes_to_keep))
# we have asserted that all nodes are caught up
# assert that view changes actually did occur
assert len(view_info) > 1
LOG.success("----------- views and primaries recorded -----------")
for view, primary in view_info.items():
LOG.success(f"view {view} - primary {primary}")
