def test_execute_benchmark_no_images_or_sources():
"""Verify the benchmark fails if no images or sources are present """
job_control = generate_test_objects.generate_default_job_control()
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkError) as benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == "No source configuration specified"
def test_execute_benchmark_envoy_source_only():
"""Verify that we detect missing NightHawk sources """
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkError) as benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == \
"No source specified to build NightHawk image"
def _setup_test(self) -> None:
"""Create the object performing the desired benchmark.
Instantiate the object performing the actual test and setup the control
objects for each test invocation.
Raises:
NotImplementedError: for tests and/or modes that are not yet implemented.
"""
current_benchmark_name = "Unspecified Benchmark"
if self._control.scavenging_benchmark:
current_benchmark_name = "Scavenging Benchmark"
job_control_list = self._generate_job_control_for_envoy_images()
for job_control in job_control_list:
benchmark = scavenging.Benchmark(job_control,
current_benchmark_name)
self._test.append(benchmark)
elif self._control.dockerized_benchmark:
current_benchmark_name = "Fully Dockerized Benchmark"
job_control_list = self._generate_job_control_for_envoy_images()
for job_control in job_control_list:
benchmark = fulldocker.Benchmark(job_control,
current_benchmark_name)
self._test.append(benchmark)
elif self._control.binary_benchmark:
current_benchmark_name = "Binary Benchmark"
# Not working with docker images here, so use custom binary-oriented job control generation
job_control_list = self._generate_job_control_for_binaries()
for job_control in job_control_list:
benchmark = binary_benchmark.Benchmark(job_control,
current_benchmark_name)
self._test.append(benchmark)
if not self._test:
raise NotImplementedError(f"No [{current_benchmark_name}] defined")
def test_execute_benchmark_no_environment(mock_benchmarks,
mock_get_source_tree):
"""Verify that we fail a benchmark if no environment is set """
job_control = generate_test_objects.generate_default_job_control()
# Add nighthawk and envoy sources
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkEnvironmentError) as \
benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == \
"No IP version is specified for the benchmark"
mock_benchmarks.assert_called()
mock_get_source_tree.assert_called()
def test_execute_benchmark(mock_benchmarks, mock_get_source_tree,
mock_run_command):
"""Verify that we fail a benchmark if no environment is set """
job_control = generate_test_objects.generate_default_job_control()
# Add nighthawk and envoy sources
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
calls = [
mock.call(
"bazel-bin/benchmarks/benchmarks "
"--log-cli-level=info -vvvv -k test_http_h1_small "
"benchmarks/", mock.ANY)
]
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
benchmark.execute_benchmark()
mock_benchmarks.assert_called()
mock_get_source_tree.assert_called()
mock_run_command.assert_has_calls(calls)