def test_retire_backup(network, args): primary, _ = network.find_primary() backup_to_retire = network.find_any_backup() network.consortium.retire_node(primary, backup_to_retire) backup_to_retire.stop() check_can_progress(primary) return network
def test_add_node_from_backup(network, args): new_node = network.create_node("local://localhost") network.join_node( new_node, args.package, args, target_node=network.find_any_backup() ) network.trust_node(new_node, args) return network
def test_forwarding_frontends(network, args): backup = network.find_any_backup() with backup.client() as c: check_commit = infra.checker.Checker(c) ack = network.consortium.get_any_active_member().ack(backup) check_commit(ack) with backup.client("user0") as c: check_commit = infra.checker.Checker(c) check = infra.checker.Checker() msg = "forwarded_msg" log_id = 123 check_commit( c.post("/app/log/private", { "id": log_id, "msg": msg }), result=True, ) check(c.get(f"/app/log/private?id={log_id}"), result={"msg": msg}) if args.package == "liblogging": escaped_query_tests(c, "request_query") return network
def test_add_node_from_backup(network, args): backup = network.find_any_backup() new_node = network.create_and_trust_node( args.package, "localhost", args, target_node=backup ) assert new_node return network
def test_retire_backup(network, args): primary, _ = network.find_primary() backup_to_retire = network.find_any_backup() network.retire_node(primary, backup_to_retire) backup_to_retire.stop() check_can_progress(primary) wait_for_reconfiguration_to_complete(network) return network
def test_primary(network, args): primary, _ = network.find_primary() with primary.client() as c: r = c.head("/node/primary") assert r.status_code == http.HTTPStatus.OK.value backup = network.find_any_backup() with backup.client() as c: r = c.head("/node/primary", allow_redirects=False) assert r.status_code == http.HTTPStatus.PERMANENT_REDIRECT.value assert (r.headers["location"] == f"https://{primary.pubhost}:{primary.pubport}/node/primary") return network
def test_retire_backup(network, args): primary, _ = network.find_primary() backup_to_retire = network.find_any_backup() network.retire_node(primary, backup_to_retire) network.wait_for_node_in_store( primary, backup_to_retire.node_id, node_status=None, timeout=3, ) backup_to_retire.stop() check_can_progress(primary) wait_for_reconfiguration_to_complete(network) return network
def test_primary(network, args, notifications_queue=None, verify=True): LOG.error(network.nodes) primary, _ = network.find_primary() LOG.error(f"PRIMARY {primary.pubhost}") with primary.client() as c: r = c.head("/node/primary") assert r.status_code == http.HTTPStatus.OK.value backup = network.find_any_backup() LOG.error(f"BACKUP {backup.pubhost}") with backup.client() as c: r = c.head("/node/primary") assert r.status_code == http.HTTPStatus.PERMANENT_REDIRECT.value assert (r.headers["location"] == f"https://{primary.pubhost}:{primary.rpc_port}/node/primary") return network
def test_primary(network, args): primary, _ = network.find_primary() with primary.client() as c: r = c.head("/node/primary") assert r.status_code == http.HTTPStatus.OK.value backup = network.find_any_backup() for interface_name in backup.host.rpc_interfaces.keys(): with backup.client(interface_name=interface_name) as c: r = c.head("/node/primary", allow_redirects=False) assert r.status_code == http.HTTPStatus.PERMANENT_REDIRECT.value primary_interface = primary.host.rpc_interfaces[interface_name] assert ( r.headers["location"] == f"https://{primary_interface.public_host}:{primary_interface.public_port}/node/primary" ) LOG.info( f'Successfully redirected to {r.headers["location"]} on primary {primary.local_node_id}' ) return network
def run(args): hosts = ["localhost", "localhost", "localhost"] LOG.info(f"setting seed to {args.seed}") random.seed(args.seed) txs = app.LoggingTxs() with infra.network.network( hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb, txs=txs ) as network: network.start_and_join(args) original_nodes = network.get_joined_nodes() view_info = {} suspend.update_view_info(network, view_info) app.test_run_txs(network=network, args=args, num_txs=TOTAL_REQUESTS) suspend.update_view_info(network, view_info) nodes_to_kill = [network.find_any_backup()] nodes_to_keep = [n for n in original_nodes if n not in nodes_to_kill] # check that a new node can catch up after all the requests late_joiner = network.create_and_trust_node(args.package, "localhost", args) nodes_to_keep.append(late_joiner) # some requests to be processed while the late joiner catches up # (no strict checking that these requests are actually being processed simultaneously with the node catchup) app.test_run_txs( network=network, args=args, num_txs=int(TOTAL_REQUESTS / 2), nodes=original_nodes, # doesn't contain late joiner verify=False, # will try to verify for late joiner and it might not be ready yet ) suspend.wait_for_late_joiner(original_nodes[0], late_joiner) # kill the old node(s) and ensure we are still making progress for backup_to_retire in nodes_to_kill: LOG.success(f"Stopping node {backup_to_retire.node_id}") backup_to_retire.stop() # check nodes are ok after we killed one off app.test_run_txs( network=network, args=args, nodes=nodes_to_keep, num_txs=len(nodes_to_keep), timeout=30, ignore_failures=True, # in the event of an early view change due to the late joiner this might # take longer than usual to complete and we don't want the test to break here ) suspend.test_suspend_nodes(network, args, nodes_to_keep) # run txs while nodes get suspended app.test_run_txs( network=network, args=args, num_txs=4 * TOTAL_REQUESTS, timeout=30, ignore_failures=True, # in the event of an early view change due to the late joiner this might # take longer than usual to complete and we don't want the test to break here ) suspend.update_view_info(network, view_info) # check nodes have resumed normal execution before shutting down app.test_run_txs(network=network, args=args, num_txs=len(nodes_to_keep)) # we have asserted that all nodes are caught up # assert that view changes actually did occur assert len(view_info) > 1 LOG.success("----------- views and primaries recorded -----------") for view, primary in view_info.items(): LOG.success(f"view {view} - primary {primary}")