def test_source_to_build_binaries(mock_cmd, mock_envoy_build,
mock_nh_bin_build, mock_nh_bench_build):
"""Validate we can build binaries from source.
Validate that sources are defined that enable us to build Envoy/Nighthawk
We do not expect the validation logic to throw an exception
"""
# create a valid configuration with a missing Envoy image
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
# Setup mock values
mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static"
mock_envoy_build.return_value = mock_envoy_path
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
benchmark.execute_benchmark()
assert benchmark._envoy_binary_path == mock_envoy_path
mock_envoy_build.assert_called_once()
mock_nh_bench_build.assert_called_once()
mock_nh_bin_build.assert_called_once()
def test_nh_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build,
mock_nh_bench_build):
"""Validate that an exception is raised which halts the benchmark execution
when the Nighthawk build fails
We expect an unhandled exception to surface from _prepare_nighthawk
"""
# Setup mock values
mock_nh_bench_build.side_effect = subprocess.CalledProcessError(1, "bar")
mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static"
mock_envoy_build.return_value = mock_envoy_path
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as build_exception:
benchmark.execute_benchmark()
assert str(build_exception.value
) == "Command 'bar' returned non-zero exit status 1."
assert not benchmark._envoy_binary_path
# We expect the nighthawk binary build to occur
# before the benchmark build fails
mock_nh_bin_build.assert_called_once()
# Raising an exception during the nighthawk build should prevent control flow
# from proceeding to build Envoy
mock_envoy_build.assert_not_called()
def test_envoy_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build,
mock_nh_bench_build):
"""Validate that an exception is raised which halts the benchmark execution
when the Envoy build fails
We expect an unhandled exception to surface from _prepare_envoy
"""
# Setup mock values
mock_envoy_build.side_effect = subprocess.CalledProcessError(1, "foo")
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as build_exception:
benchmark.execute_benchmark()
assert str(build_exception.value
) == "Command 'foo' returned non-zero exit status 1."
assert not benchmark._envoy_binary_path
# We expect the nighthawk build to occur before the Envoy build fails
mock_nh_bench_build.assert_called_once()
mock_nh_bin_build.assert_called_once()
def test_no_sources():
"""Test benchmark validation logic.
We expect the validation logic to throw an exception,
since no sources are present
"""
# create a valid configuration with no images
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark should throw an exception
with pytest.raises(binary_benchmark.BinaryBenchmarkError) \
as validation_error:
benchmark.execute_benchmark()
assert str(validation_error.value) == "No source configuration specified"
def test_no_valid_envoy_binary(mock_nh_bin_build, mock_nh_bench_build):
"""Validate that we fail when Envoy sources are not present,
and no binary is specified.
We expect an unhandled exception to surface from _prepare_envoy()
"""
# create a valid configuration with a missing both NightHawk container images
job_control = proto_control.JobControl(remote=False, binary_benchmark=True)
job_control.environment.variables['ENVOY_PATH'] = '/dev/null/foo'
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"ENVOY_PATH environment variable specified, but invalid"
def _setup_test(self) -> None:
"""Create the object performing the desired benchmark.
Instantiate the object performing the actual test and setup the control
objects for each test invocation.
Raises:
NotImplementedError: for tests and/or modes that are not yet implemented.
"""
current_benchmark_name = "Unspecified Benchmark"
if self._control.scavenging_benchmark:
current_benchmark_name = "Scavenging Benchmark"
job_control_list = self._generate_job_control_for_envoy_images()
for job_control in job_control_list:
benchmark = scavenging.Benchmark(job_control,
current_benchmark_name)
self._test.append(benchmark)
elif self._control.dockerized_benchmark:
current_benchmark_name = "Fully Dockerized Benchmark"
job_control_list = self._generate_job_control_for_envoy_images()
for job_control in job_control_list:
benchmark = fulldocker.Benchmark(job_control,
current_benchmark_name)
self._test.append(benchmark)
elif self._control.binary_benchmark:
current_benchmark_name = "Binary Benchmark"
# Not working with docker images here, so use custom binary-oriented job control generation
job_control_list = self._generate_job_control_for_binaries()
for job_control in job_control_list:
benchmark = binary_benchmark.Benchmark(job_control,
current_benchmark_name)
self._test.append(benchmark)
if not self._test:
raise NotImplementedError(f"No [{current_benchmark_name}] defined")
def test_no_source_to_build_nh():
"""Validate that we fail the entire process in the absence of NH sources
Validate that even if Envoy sources are specified, the absence of Nighthawk
sources will cause the program to fail.
We expect the validation logic to throw an exception
"""
# create a valid configuration with a missing NightHawk container image
job_control = proto_control.JobControl(remote=False, binary_benchmark=True)
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark should throw an exception
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build Nighthawk"