def test_http_h2(http_test_server_fixture):
"""Test h2 over plain http.
Runs the CLI configured to use h2c against our test server, and sanity
checks statistics from both client and server.
"""
parsed_json, _ = http_test_server_fixture.runNighthawkClient([
"--h2",
http_test_server_fixture.getTestServerRootUri(), "--max-active-requests", "1", "--duration",
"100", "--termination-predicate", "benchmark.http_2xx:24", "--rps", "100"
])
counters = http_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25)
asserts.assertCounterEqual(counters, "upstream_cx_http2_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_rx_bytes_total", 1030)
asserts.assertCounterEqual(counters, "upstream_cx_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_tx_bytes_total", 403)
asserts.assertCounterEqual(counters, "upstream_rq_pending_total", 1)
asserts.assertCounterEqual(counters, "upstream_rq_total", 25)
asserts.assertCounterEqual(counters, "default.total_match_count", 1)
asserts.assertEqual(len(counters), 12)
def test_http_request_release_timing(http_test_server_fixture,
qps_parameterization_fixture,
duration_parameterization_fixture):
"""Test latency-sample-, query- and reply- counts in various configurations."""
for concurrency in [1, 2]:
parsed_json, _ = http_test_server_fixture.runNighthawkClient([
http_test_server_fixture.getTestServerRootUri(), "--duration",
str(duration_parameterization_fixture), "--rps",
str(qps_parameterization_fixture), "--concurrency",
str(concurrency)
])
total_requests = qps_parameterization_fixture * concurrency * duration_parameterization_fixture
global_histograms = http_test_server_fixture.getNighthawkGlobalHistogramsbyIdFromJson(
parsed_json)
counters = http_test_server_fixture.getNighthawkCounterMapFromJson(
parsed_json)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.request_to_response"]
["count"]), total_requests)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.queue_to_connect"]
["count"]), total_requests)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.latency_2xx"]
["count"]), total_requests)
asserts.assertCounterEqual(counters, "benchmark.http_2xx",
(total_requests))
def test_http_h1_response_header_latency_tracking(http_test_server_fixture,
server_config):
"""Test emission and tracking of response header latencies.
Run the CLI configured to track latencies delivered by response header from the test-server.
Ensure that the origin_latency_statistic histogram receives the correct number of inputs.
"""
parsed_json, _ = http_test_server_fixture.runNighthawkClient([
http_test_server_fixture.getTestServerRootUri(), "--connections", "1",
"--rps", "100", "--duration", "100", "--termination-predicate",
"benchmark.http_2xx:99", "--latency-response-header-name",
"x-origin-request-receipt-delta"
])
global_histograms = http_test_server_fixture.getNighthawkGlobalHistogramsbyIdFromJson(
parsed_json)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.latency_2xx"]["count"]),
100)
# Verify behavior is correct both with and without the timing filter enabled.
expected_histogram_count = 99 if "nighthawk_track_timings.yaml" in server_config else 0
asserts.assertEqual(
int(global_histograms["benchmark_http_client.origin_latency_statistic"]
["count"]), expected_histogram_count)
def test_h1_pool_strategy(http_test_server_fixture):
"""Test connection re-use strategies of the http 1 connection pool.
Test that with the "mru" strategy only the first created connection gets to send requests.
Then, with the "lru" strategy, we expect the other connection to be used as well.
"""
def countLogLinesWithSubstring(logs, substring):
return len(
[line for line in logs.split(os.linesep) if substring in line])
_, logs = http_test_server_fixture.runNighthawkClient([
"--rps 5", "-v", "trace", "--connections", "2",
"--prefetch-connections",
"--experimental-h1-connection-reuse-strategy", "mru",
"--termination-predicate", "benchmark.http_2xx:4",
http_test_server_fixture.getTestServerRootUri()
])
asserts.assertNotIn("[C1] message complete", logs)
asserts.assertEqual(
countLogLinesWithSubstring(logs, "[C0] message complete"), 10)
requests = 12
connections = 3
_, logs = http_test_server_fixture.runNighthawkClient([
"--rps", "5", "-v trace", "--connections",
str(connections), "--prefetch-connections",
"--experimental-h1-connection-reuse-strategy", "lru",
"--termination-predicate",
"benchmark.http_2xx:%d" % (requests - 1),
http_test_server_fixture.getTestServerRootUri()
])
for i in range(1, connections):
line_count = countLogLinesWithSubstring(logs,
"[C%d] message complete" % i)
strict_count = (requests / connections) * 2
asserts.assertBetweenInclusive(line_count, strict_count, strict_count)
def test_https_h1(https_test_server_fixture):
"""Test h1 over https.
Runs the CLI configured to use HTTP/1 over https against our test server, and sanity
checks statistics from both client and server.
"""
parsed_json, _ = https_test_server_fixture.runNighthawkClient([
https_test_server_fixture.getTestServerRootUri(), "--connections", "1",
"--rps", "100", "--duration", "100", "--termination-predicate",
"benchmark.http_2xx:24"
])
counters = https_test_server_fixture.getNighthawkCounterMapFromJson(
parsed_json)
asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25)
asserts.assertCounterEqual(counters, "upstream_cx_rx_bytes_total", 3400)
# It is possible that the # of upstream_cx > # of backend connections for H1 as new connections
# will spawn if the existing clients cannot keep up with the RPS.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_tx_bytes_total",
500)
asserts.assertCounterGreaterEqual(counters, "upstream_rq_pending_total", 1)
asserts.assertCounterEqual(counters, "upstream_rq_total", 25)
asserts.assertCounterEqual(counters,
"ssl.ciphers.ECDHE-RSA-AES128-GCM-SHA256", 1)
asserts.assertCounterEqual(counters, "ssl.curves.X25519", 1)
asserts.assertCounterEqual(counters, "ssl.handshake", 1)
asserts.assertCounterEqual(counters, "ssl.sigalgs.rsa_pss_rsae_sha256", 1)
asserts.assertCounterEqual(counters, "ssl.versions.TLSv1.2", 1)
asserts.assertCounterEqual(counters, "default.total_match_count", 1)
asserts.assertEqual(len(counters), 17)
server_stats = https_test_server_fixture.getTestServerStatisticsJson()
asserts.assertEqual(
https_test_server_fixture.getServerStatFromJson(
server_stats, "http.ingress_http.downstream_rq_2xx"), 25)
def test_client_bad_arg():
"""Test that passing bad arguments behaves as expected."""
(exit_code, output) = _run_client_with_args("127.0.0.1 --foo")
asserts.assertEqual(exit_code, 1)
asserts.assertIn("PARSE ERROR: Argument: --foo", output)
def test_client_help():
"""Test that passing --help behaves as expected."""
(exit_code, output) = _run_client_with_args("--help")
asserts.assertEqual(exit_code, 0)
asserts.assertIn("USAGE", output)
def test_http_h1(http_test_server_fixture):
"""Test http1 over plain http.
Runs the CLI configured to use plain HTTP/1 against our test server, and sanity
checks statistics from both client and server.
"""
parsed_json, _ = http_test_server_fixture.runNighthawkClient([
http_test_server_fixture.getTestServerRootUri(), "--duration", "100",
"--termination-predicate", "benchmark.http_2xx:24"
])
counters = http_test_server_fixture.getNighthawkCounterMapFromJson(
parsed_json)
asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25)
asserts.assertCounterEqual(counters, "upstream_cx_http1_total", 1)
asserts.assertCounterEqual(counters, "upstream_cx_rx_bytes_total", 3400)
asserts.assertCounterEqual(counters, "upstream_cx_total", 1)
asserts.assertCounterEqual(
counters, "upstream_cx_tx_bytes_total", 1375
if http_test_server_fixture.ip_version == IpVersion.IPV6 else 1450)
asserts.assertCounterEqual(counters, "upstream_rq_pending_total", 1)
asserts.assertCounterEqual(counters, "upstream_rq_total", 25)
asserts.assertCounterEqual(counters, "default.total_match_count", 1)
global_histograms = http_test_server_fixture.getNighthawkGlobalHistogramsbyIdFromJson(
parsed_json)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["count"]), 25)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["count"]), 25)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["raw_mean"]), 10)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["raw_mean"]), 97)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["raw_min"]), 10)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["raw_min"]), 97)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["raw_max"]), 10)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["raw_max"]), 97)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]
["raw_pstdev"]), 0)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]
["raw_pstdev"]), 0)
asserts.assertEqual(len(counters), 12)
def test_http_h1(http_test_server_fixture):
"""Test http1 over plain http.
Runs the CLI configured to use plain HTTP/1 against our test server, and sanity
checks statistics from both client and server.
"""
parsed_json, _ = http_test_server_fixture.runNighthawkClient([
http_test_server_fixture.getTestServerRootUri(), "--duration", "100",
"--termination-predicate", "benchmark.http_2xx:24"
])
counters = http_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json)
asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25)
asserts.assertCounterEqual(counters, "upstream_cx_rx_bytes_total", 3400)
# It is possible that the # of upstream_cx > # of backend connections for H1
# as new connections will spawn if the existing clients cannot keep up with the RPS.
asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_total", 1)
asserts.assertCounterGreaterEqual(counters, "upstream_cx_tx_bytes_total", 500)
asserts.assertCounterGreaterEqual(counters, "upstream_rq_pending_total", 1)
asserts.assertCounterEqual(counters, "upstream_rq_total", 25)
asserts.assertCounterEqual(counters, "default.total_match_count", 1)
global_histograms = http_test_server_fixture.getNighthawkGlobalHistogramsbyIdFromJson(parsed_json)
asserts.assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["count"]),
25)
asserts.assertEqual(int(global_histograms["benchmark_http_client.response_header_size"]["count"]),
25)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]["raw_mean"]), 10)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]["raw_mean"]), 97)
asserts.assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["raw_min"]),
10)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]["raw_min"]), 97)
asserts.assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["raw_max"]),
10)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]["raw_max"]), 97)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_body_size"]["raw_pstdev"]), 0)
asserts.assertEqual(
int(global_histograms["benchmark_http_client.response_header_size"]["raw_pstdev"]), 0)
asserts.assertGreaterEqual(len(counters), 12)
def test_grpc_service_nonexisting_listener_address():
"""Test that the gRPC service behaves as expected when an address is passed that it can't listen to."""
(exit_code, output) = _run_service_with_args("--listen 1.1.1.1:1")
asserts.assertEqual(exit_code, 1)
asserts.assertIn("Failure: Could not start the grpc service", output)
def test_grpc_service_bad_arguments():
"""Test that the gRPC service behaves as expected with bad cli arguments."""
(exit_code, output) = _run_service_with_args("--foo")
asserts.assertEqual(exit_code, 1)
asserts.assertIn("PARSE ERROR: Argument: --foo", output)
def test_grpc_service_help():
"""Test that the gRPC service behaves as expected when --help is passed."""
(exit_code, output) = _run_service_with_args("--help")
asserts.assertEqual(exit_code, 0)
asserts.assertIn("USAGE", output)
def test_output_transform_bad_arguments():
"""Test that the output transform binary behaves as expected when bad arguments are passed."""
(exit_code, output) = _run_output_transform_with_args("--foo")
asserts.assertEqual(exit_code, 1)
asserts.assertIn("PARSE ERROR: Argument: --foo", output)
def test_output_transform_help():
"""Test that the output transform binary behaves as expected when --help is passed."""
(exit_code, output) = _run_output_transform_with_args("--help")
asserts.assertEqual(exit_code, 0)
asserts.assertIn("USAGE", output)
