def test_http_h2_mini_stress_test_with_client_side_queueing(
http_test_server_fixture):
"""Run a max rps test with the h2 pool against our test server, using a small client-side queue."""
counters = _mini_stress_test(http_test_server_fixture, [
http_test_server_fixture.getTestServerRootUri(), "--rps", "999999",
"--max-pending-requests", "10", "--h2", "--max-active-requests", "1",
"--connections", "1", "--duration", "100", "--termination-predicate",
"benchmark.http_2xx:99", "--simple-warmup"
])
asserts.assertCounterEqual(counters, "upstream_rq_pending_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_rq_pending_overflow",
10)
def check_upload_expectations(fixture, parsed_json,
expected_transmitted_bytes,
expected_received_bytes):
counters = fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterGreaterEqual(counters,
"upstream_cx_tx_bytes_total",
expected_transmitted_bytes)
server_stats = fixture.getTestServerStatisticsJson()
# Server side expectations start failing with larger upload sizes
asserts.assertGreaterEqual(
fixture.getServerStatFromJson(
server_stats,
"http.ingress_http.downstream_cx_rx_bytes_total"),
expected_received_bytes)
def test_https_h1(https_test_server_fixture):
"""Test h1 over https.
Runs the CLI configured to use HTTP/1 over https against our test server, and sanity
checks statistics from both client and server.
"""
parsed_json, _ = https_test_server_fixture.runNighthawkClient([
https_test_server_fixture.getTestServerRootUri(), "--connections", "1", "--rps", "100",
"--duration", "100", "--termination-predicate", "benchmark.http_2xx:24"
])
counters = https_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25)
asserts.assertCounterEqual(counters, "upstream_cx_rx_bytes_total", 3400)
# It is possible that the # of upstream_cx > # of backend connections for H1 as new connections
# will spawn if the existing clients cannot keep up with the RPS.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_tx_bytes_total", 500)
asserts.assertCounterGreaterEqual(counters, "upstream_rq_pending_total", 1)
asserts.assertCounterEqual(counters, "upstream_rq_total", 25)
asserts.assertCounterEqual(counters, "ssl.ciphers.ECDHE-RSA-AES128-GCM-SHA256", 1)
asserts.assertCounterEqual(counters, "ssl.curves.X25519", 1)
asserts.assertCounterEqual(counters, "ssl.handshake", 1)
asserts.assertCounterEqual(counters, "ssl.sigalgs.rsa_pss_rsae_sha256", 1)
asserts.assertCounterEqual(counters, "ssl.versions.TLSv1.2", 1)
asserts.assertCounterEqual(counters, "default.total_match_count", 1)
asserts.assertEqual(len(counters), 17)
server_stats = https_test_server_fixture.getTestServerStatisticsJson()
asserts.assertEqual(
https_test_server_fixture.getServerStatFromJson(server_stats,
"http.ingress_http.downstream_rq_2xx"), 25)
def test_https_h2_sni(https_test_server_fixture):
"""Tests that SNI indication works on https/h1."""
# Verify success when we set the right host
parsed_json, _ = https_test_server_fixture.runNighthawkClient([
https_test_server_fixture.getTestServerRootUri(), "--rps", "100", "--duration", "100",
"--termination-predicate", "benchmark.http_2xx:2", "--request-header", ":authority: sni.com",
"--h2"
])
counters = https_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterGreaterEqual(counters, "benchmark.http_2xx", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http2_total", 1)
asserts.assertCounterEqual(counters, "ssl.handshake", 1)
# Verify success when we set the right host
parsed_json, _ = https_test_server_fixture.runNighthawkClient([
https_test_server_fixture.getTestServerRootUri(), "--rps", "100", "--duration", "100",
"--termination-predicate", "benchmark.http_2xx:2", "--request-header", "host: sni.com", "--h2"
])
counters = https_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterGreaterEqual(counters, "benchmark.http_2xx", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http2_total", 1)
asserts.assertCounterEqual(counters, "ssl.handshake", 1)
# Verify failure when we set no host (will get plain http)
parsed_json, _ = https_test_server_fixture.runNighthawkClient([
https_test_server_fixture.getTestServerRootUri(), "--rps", "100", "--duration", "100", "--h2"
],
expect_failure=True)
# Verify failure when we provide both host and :authority: (will get plain http)
parsed_json, _ = https_test_server_fixture.runNighthawkClient([
https_test_server_fixture.getTestServerRootUri(), "--rps", "100", "--duration", "100", "--h2",
"--request-header", "host: sni.com", "--request-header", ":authority: sni.com"
],
expect_failure=True)
def test_http_h1(http_test_server_fixture):
"""Test http1 over plain http.
Runs the CLI configured to use plain HTTP/1 against our test server, and sanity
checks statistics from both client and server.
"""
parsed_json, _ = http_test_server_fixture.runNighthawkClient([
http_test_server_fixture.getTestServerRootUri(), "--duration", "100",
"--termination-predicate", "benchmark.http_2xx:24"
])
counters = http_test_server_fixture.getNighthawkCounterMapFromJson(
parsed_json)
asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25)
asserts.assertCounterEqual(counters, "upstream_cx_rx_bytes_total", 3400)
# It is possible that the # of upstream_cx > # of backend connections for H1
# as new connections will spawn if the existing clients cannot keep up with the RPS.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_tx_bytes_total",
500)
asserts.assertCounterGreaterEqual(counters, "upstream_rq_pending_total", 1)
asserts.assertCounterEqual(counters, "upstream_rq_total", 25)
asserts.assertCounterEqual(counters, "default.total_match_count", 1)
global_histograms = http_test_server_fixture.getNighthawkGlobalHistogramsbyIdFromJson(
parsed_json)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["count"]), 25)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["count"]), 25)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["raw_mean"]), 10)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["raw_mean"]), 97)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["raw_min"]), 10)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["raw_min"]), 97)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["raw_max"]), 10)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["raw_max"]), 97)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["raw_pstdev"]), 0)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["raw_pstdev"]), 0)
asserts.assertGreaterEqual(len(counters), 12)
def test_cancellation_with_infinite_duration(http_test_server_fixture):
"""Test that we can use signals to cancel execution."""
args = [
http_test_server_fixture.nighthawk_client_path, "--concurrency", "2",
http_test_server_fixture.getTestServerRootUri(), "--no-duration", "--output-format", "json"
]
client_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Thread(target=(lambda: _send_sigterm(client_process))).start()
stdout, stderr = client_process.communicate()
client_process.wait()
output = stdout.decode('utf-8')
asserts.assertEqual(client_process.returncode, 0)
parsed_json = json.loads(output)
counters = http_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterEqual(counters, "graceful_stop_requested", 2)
asserts.assertCounterGreaterEqual(counters, "benchmark.http_2xx", 1)
def test_http_concurrency(http_test_server_fixture):
"""Test that concurrency acts like a multiplier."""
parsed_json, _ = http_test_server_fixture.runNighthawkClient([
"--concurrency 4 --rps 100 --connections 1", "--duration", "100", "--termination-predicate",
"benchmark.http_2xx:24",
http_test_server_fixture.getTestServerRootUri()
])
counters = http_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
# Quite a loose expectation, but this may fluctuate depending on server load.
# Ideally we'd see 4 workers * 5 rps * 5s = 100 requests total
asserts.assertCounterEqual(counters, "benchmark.http_2xx", 100)
# Assert that we at least have 1 connection for each event loop (1*4). It is possible that the # of
# upstream_cx > # of backend connections for H1 as new connections will spawn if the existing clients
# cannot keep up with the RPS.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 4)
def test_drain(https_test_server_fixture):
"""Test that the pool drain timeout is effective, and we terminate in a timely fashion.
Sets up the test server to delay replies 100 seconds. Our execution will only last 30 seconds, so we
expect to observe no replies. Termination should be cut short by the drain timeout, which means
that we should have results in approximately execution duration + drain timeout = 35 seconds.
(the pool drain timeout is hard coded to 5 seconds as of writing this).
If drain timeout is reached, a message will be logged to the user.
"""
parsed_json, logs = https_test_server_fixture.runNighthawkClient([
https_test_server_fixture.getTestServerRootUri(), "--rps", "100", "--duration", "20",
"--request-header", "x-nighthawk-test-server-config: {static_delay: \"100s\"}"
])
counters = https_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 1)
asserts.assertNotIn("benchmark.http_2xx", counters)
asserts.assertIn("Wait for the connection pool drain timed out, proceeding to hard shutdown",
logs)
def test_https_h2_multiple_connections(https_test_server_fixture):
"""Test that the experimental h2 pool uses multiple connections.
The burst we send ensures we will need 10 connections right away, as we
limit max active streams per connection to 1 by setting the experimental
flag to use multiple h2 connections.
"""
parsed_json, _ = https_test_server_fixture.runNighthawkClient([
"--h2",
https_test_server_fixture.getTestServerRootUri(), "--rps", "100", "--duration", "100",
"--termination-predicate", "benchmark.http_2xx:99", "--max-active-requests", "10",
"--max-pending-requests", "10", "--max-concurrent-streams", "1", "--burst-size", "10"
])
counters = https_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterGreaterEqual(counters, "benchmark.http_2xx", 100)
# Empirical observation shows we may end up creating more then 10 connections.
# This is stock Envoy h/2 pool behavior.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http2_total", 10)
def test_drain(https_test_server_fixture):
"""Test that the pool drain timeout is effective, and we terminate in a timely fashion.
Sets up the test server to delay replies 100 seconds. Our execution will only last 3 seconds, so we
expect to observe no replies. Termination should be cut short by the drain timeout, which means
that we should have results in approximately execution duration + drain timeout = 8 seconds.
(the pool drain timeout is hard coded to 5 seconds as of writing this).
"""
t0 = time.time()
parsed_json, _ = https_test_server_fixture.runNighthawkClient([
https_test_server_fixture.getTestServerRootUri(), "--rps", "100", "--duration", "3",
"--request-header", "x-nighthawk-test-server-config: {static_delay: \"100s\"}"
])
t1 = time.time()
time_delta = t1 - t0
counters = https_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
assert time_delta < 40 # *lots* of slack to avoid failure in slow CI executions.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 1)
asserts.assertNotIn("benchmark.http_2xx", counters)
def test_http_request_release_timing(http_test_server_fixture,
qps_parameterization_fixture,
duration_parameterization_fixture):
"""Test latency-sample-, query- and reply- counts in various configurations."""
for concurrency in [1, 2]:
parsed_json, _ = http_test_server_fixture.runNighthawkClient([
http_test_server_fixture.getTestServerRootUri(), "--duration",
str(duration_parameterization_fixture), "--rps",
str(qps_parameterization_fixture), "--concurrency",
str(concurrency)
])
global_histograms = http_test_server_fixture.getNighthawkGlobalHistogramsbyIdFromJson(
parsed_json)
counters = http_test_server_fixture.getNighthawkCounterMapFromJson(
parsed_json)
global_result = http_test_server_fixture.getGlobalResults(parsed_json)
actual_duration = utility.get_execution_duration_from_global_result_json(
global_result)
# Ensure Nighthawk managed to execute for at least some time.
assert actual_duration >= 1
# The actual duration is a float, flooring if here allows us to use
# the GreaterEqual matchers below.
total_requests = qps_parameterization_fixture * concurrency * math.floor(
actual_duration)
asserts.assertGreaterEqual(
int(global_histograms["benchmark_http_client.request_to_response"]
["count"]), total_requests)
asserts.assertGreaterEqual(
int(global_histograms["benchmark_http_client.queue_to_connect"]
["count"]), total_requests)
asserts.assertGreaterEqual(
int(global_histograms["benchmark_http_client.latency_2xx"]
["count"]), total_requests)
asserts.assertCounterGreaterEqual(counters, "benchmark.http_2xx",
(total_requests))
# Give system resources some time to recover after the last execution.
time.sleep(2)
def test_http_h2(http_test_server_fixture):
"""Test h2 over plain http.
Runs the CLI configured to use h2c against our test server, and sanity
checks statistics from both client and server.
"""
parsed_json, _ = http_test_server_fixture.runNighthawkClient([
"--h2",
http_test_server_fixture.getTestServerRootUri(), "--max-active-requests", "1", "--duration",
"100", "--termination-predicate", "benchmark.http_2xx:24", "--rps", "100"
])
counters = http_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25)
asserts.assertCounterEqual(counters, "upstream_cx_http2_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_rx_bytes_total", 1030)
asserts.assertCounterEqual(counters, "upstream_cx_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_tx_bytes_total", 403)
asserts.assertCounterEqual(counters, "upstream_rq_pending_total", 1)
asserts.assertCounterEqual(counters, "upstream_rq_total", 25)
asserts.assertCounterEqual(counters, "default.total_match_count", 1)
asserts.assertGreaterEqual(len(counters), 12)
def test_multiple_backends_https_h1(multi_https_test_server_fixture):
"""Test that we can load-test multiple backends on https.
Runs the CLI configured to use HTTP/1 with TLS against multiple test servers, and sanity
checks statistics from both client and server.
"""
nighthawk_client_args = [
"--multi-target-use-https", "--multi-target-path", "/", "--duration", "100",
"--termination-predicate", "benchmark.http_2xx:24"
]
for uri in multi_https_test_server_fixture.getAllTestServerRootUris():
nighthawk_client_args.append("--multi-target-endpoint")
nighthawk_client_args.append(uri.replace("https://", "").replace("/", ""))
parsed_json, stderr = multi_https_test_server_fixture.runNighthawkClient(nighthawk_client_args)
counters = multi_https_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25)
asserts.assertCounterGreater(counters, "upstream_cx_rx_bytes_total", 0)
# Assert that we at least have 1 connection per backend. It is possible that
# the # of upstream_cx > # of backend connections for H1 as new connections
# will spawn if the existing clients cannot keep up with the RPS.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 3)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_total", 3)
asserts.assertCounterGreaterEqual(counters, "upstream_rq_pending_total", 3)
asserts.assertCounterGreater(counters, "upstream_cx_tx_bytes_total", 0)
asserts.assertCounterEqual(counters, "upstream_rq_total", 25)
asserts.assertCounterEqual(counters, "default.total_match_count", 3)
for parsed_server_json in multi_https_test_server_fixture.getAllTestServerStatisticsJsons():
single_2xx = multi_https_test_server_fixture.getServerStatFromJson(
parsed_server_json, "http.ingress_http.downstream_rq_2xx")
# Confirm that each backend receives some traffic
asserts.assertGreaterEqual(single_2xx, 1)
def test_https_h1_sni(https_test_server_fixture):
"""Test that SNI indication works on https/h1."""
# Verify success when we set the right host
parsed_json, _ = https_test_server_fixture.runNighthawkClient([
https_test_server_fixture.getTestServerRootUri(), "--rps", "100",
"--duration", "100", "--termination-predicate", "benchmark.http_2xx:2",
"--request-header", "host: sni.com"
])
counters = https_test_server_fixture.getNighthawkCounterMapFromJson(
parsed_json)
asserts.assertCounterGreaterEqual(counters, "benchmark.http_2xx", 1)
# It is possible that the # of upstream_cx > # of backend connections for H1
# as new connections will spawn if the existing clients cannot keep up with the RPS.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 1)
asserts.assertCounterGreaterEqual(counters, "ssl.handshake", 1)
# Verify failure when we set no host (will get plain http)
parsed_json, _ = https_test_server_fixture.runNighthawkClient(
[
https_test_server_fixture.getTestServerRootUri(), "--rps", "20",
"--duration", "100"
],
expect_failure=True)
# Verify success when we use plain http and don't request the sni host
parsed_json, _ = https_test_server_fixture.runNighthawkClient(
[
https_test_server_fixture.getTestServerRootUri().replace(
"https://", "http://"), "--rps", "100", "--duration", "20",
"--termination-predicate", "benchmark.http_2xx:2"
],
expect_failure=False)
counters = https_test_server_fixture.getNighthawkCounterMapFromJson(
parsed_json)
asserts.assertCounterGreaterEqual(counters, "benchmark.http_2xx", 1)
# It is possible that the # of upstream_cx > # of backend connections for H1
# as new connections will spawn if the existing clients cannot keep up with the RPS.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 1)
asserts.assertNotIn("ssl.handshake", counters)
def _run_benchmark(fixture,
rps=1000,
duration=30,
max_connections=1,
max_active_requests=100,
request_body_size=0,
response_size=1024,
concurrency=1):
if hasattr(fixture, "proxy_server"):
assert (fixture.proxy_server.enableCpuProfiler())
assert (fixture.test_server.enableCpuProfiler())
args = [
fixture.getTestServerRootUri(), "--rps",
str(rps), "--duration",
str(duration), "--connections",
str(max_connections), "--max-active-requests",
str(max_active_requests), "--concurrency",
str(concurrency), "--request-header",
"x-nighthawk-test-server-config:{response_body_size:%s}" %
response_size, "--experimental-h1-connection-reuse-strategy", "lru",
"--prefetch-connections"
]
if request_body_size > 0:
args.append("--request-body-size")
args.append(str(request_body_size))
parsed_json, _ = fixture.runNighthawkClient(args)
counters = fixture.getNighthawkCounterMapFromJson(parsed_json)
response_count = counters["benchmark.http_2xx"]
request_count = counters["upstream_rq_total"]
connection_counter = "upstream_cx_http1_total"
# Some arbitrary sanity checks
asserts.assertCounterGreaterEqual(counters, "benchmark.http_2xx",
(concurrency * rps * duration) * 0.99)
asserts.assertGreater(counters["upstream_cx_rx_bytes_total"],
response_count * response_size)
asserts.assertGreater(counters["upstream_cx_tx_bytes_total"],
request_count * request_body_size)
asserts.assertCounterEqual(counters, connection_counter,
concurrency * max_connections)
# Could potentially set thresholds on acceptable latency here.
# dump human readable output to logs
json_as_string = json.dumps(parsed_json)
human_output = fixture.transformNighthawkJson(json_as_string, "human")
logging.info(human_output)
with open(os.path.join(fixture.test_server.tmpdir, "nighthawk-human.txt"),
"w") as f:
f.write(human_output)
with open(os.path.join(fixture.test_server.tmpdir, "nighthawk.json"),
"w") as f:
f.write(json_as_string)
with open(os.path.join(fixture.test_server.tmpdir, "nighthawk.yaml"),
"w") as f:
f.write(fixture.transformNighthawkJson(json_as_string, "yaml"))
with open(os.path.join(fixture.test_server.tmpdir, "fortio.json"),
"w") as f:
f.write(fixture.transformNighthawkJson(json_as_string, "fortio"))
with open(os.path.join(fixture.test_server.tmpdir, "server_version.txt"),
"w") as f:
f.write(fixture.test_server.getCliVersionString())
if hasattr(fixture, "proxy_server"):
with open(
os.path.join(fixture.test_server.tmpdir, "proxy_version.txt"),
"w") as f:
f.write(fixture.proxy_server.getCliVersionString())
r = runfiles.Create()
copyfile(
r.Rlocation("nighthawk/benchmarks/test/templates/simple_plot.html"),
os.path.join(fixture.test_server.tmpdir, "simple_plot.html"))