def test_images_only_config():
"""Test benchmark validation logic."""
# create a valid configuration defining images only for benchmark
job_control = _generate_default_job_control()
_generate_images(job_control)
_generate_environment(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
run_parameters = docker_image.DockerRunParameters(
command=['./benchmarks', '--log-cli-level=info', '-vvvv'],
environment=mock.ANY,
volumes=mock.ANY,
network_mode='host',
tty=True
)
# Calling execute_benchmark shoud not throw an exception
# Mock the container invocation so that we don't try to pull an image
with mock.patch('src.lib.benchmark.base_benchmark.BaseBenchmark.run_image') \
as docker_mock:
benchmark.execute_benchmark()
docker_mock.assert_called_once_with(
'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag',
run_parameters)
def test_execute_benchmark_with_envoy_source(mock_run_image):
"""Validate that if sources are defined that enable us to build the Envoy
image we do not throw an exception.
"""
mock_run_image.return_value = b'benchmark_http_client output'
# create a valid configuration with a missing Envoy image
job_control = generate_test_objects.generate_default_job_control()
images = generate_test_objects.generate_images(job_control)
images.envoy_image = ""
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
run_parameters = docker_image.DockerRunParameters(
command=['./benchmarks', '--log-cli-level=info', '-vvvv'],
environment=mock.ANY,
volumes=mock.ANY,
network_mode='host',
tty=True)
benchmark.execute_benchmark()
mock_run_image.assert_called_once_with(
'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag',
run_parameters)
def test_execute_benchmark_using_images_only(mock_run_image):
"""Validate we execute the benchmark with only images specified."""
mock_run_image.return_value = b'benchmark_http_client output'
# create a valid configuration defining images only for benchmark
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_images(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
run_parameters = docker_image.DockerRunParameters(
command=['./benchmarks', '--log-cli-level=info', '-vvvv'],
environment=mock.ANY,
volumes=mock.ANY,
network_mode='host',
tty=True)
# Calling execute_benchmark shoud not throw an exception
benchmark.execute_benchmark()
mock_run_image.assert_called_once_with(
'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag',
run_parameters)
def test_no_source_to_build_nh2():
"""Validate that we fail to build nighthawk without sources.
Validate that no sources are defined that enable us to build the missing
NightHawk binary image.
We expect the validation logic to throw an exception
"""
# create a valid configuration with a missing both NightHawk container images
job_control = proto_control.JobControl(
remote=False,
scavenging_benchmark=True
)
images = _generate_images(job_control)
images.nighthawk_binary_image = ""
images.nighthawk_benchmark_image = ""
# Generate a default Envoy source object. Values aren't really checked at
# this stage since we have a missing Envoy image, nighthawk source validation
# should fail.
_generate_envoy_source(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark shoud not throw an exception
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build unspecified NightHawk image"
def test_no_source_to_build_envoy():
"""Validate that we fail to build images without sources.
Validate that no sources are present that enable us to build the missing
Envoy image
We expect the validation logic to throw an exception
"""
# create a configuration with a missing Envoy image
job_control = proto_control.JobControl(
remote=False,
scavenging_benchmark=True
)
images = _generate_images(job_control)
images.envoy_image = ""
# Denote that the soure is for nighthawk. Values aren't really checked at
# this stage since we have a missing Envoy image and a nighthawk source
# validation should fail.
envoy_source = _generate_envoy_source(job_control)
envoy_source.identity = envoy_source.SRCID_NIGHTHAWK
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark shoud not throw an exception
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build unspecified Envoy image"
def _setup_test(self) -> None:
"""Create the object performing the desired benchmark.
Instantiate the object performing the actual test and setup the control
objects for each test invocation.
Raises:
NotImplementedError: for tests and/or modes that are not yet implemented.
"""
current_benchmark_name = "Unspecified Benchmark"
if self._control.scavenging_benchmark:
current_benchmark_name = "Scavenging Benchmark"
job_control_list = self._generate_job_control_for_envoy_images()
for job_control in job_control_list:
benchmark = scavenging.Benchmark(job_control,
current_benchmark_name)
self._test.append(benchmark)
elif self._control.dockerized_benchmark:
current_benchmark_name = "Fully Dockerized Benchmark"
job_control_list = self._generate_job_control_for_envoy_images()
for job_control in job_control_list:
benchmark = fulldocker.Benchmark(job_control,
current_benchmark_name)
self._test.append(benchmark)
elif self._control.binary_benchmark:
current_benchmark_name = "Binary Benchmark"
# Not working with docker images here, so use custom binary-oriented job control generation
job_control_list = self._generate_job_control_for_binaries()
for job_control in job_control_list:
benchmark = binary_benchmark.Benchmark(job_control,
current_benchmark_name)
self._test.append(benchmark)
if not self._test:
raise NotImplementedError(f"No [{current_benchmark_name}] defined")
def test_execute_benchmark_no_image_or_sources():
"""Verify that the validation logic raises an exception since we are unable to
build a required Envoy image.
"""
# create a valid configuration with a missing Envoy image
job_control = generate_test_objects.generate_default_job_control()
images = generate_test_objects.generate_images(job_control)
images.envoy_image = ""
generate_test_objects.generate_environment(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark shoud not throw an exception
with pytest.raises(base_benchmark.BenchmarkError) \
as validation_error:
benchmark.execute_benchmark()
assert str(validation_error.value) == "No source configuration specified"
def test_execute_benchmark_missing_nighthawk_binary_image():
"""Validate that no sources are defined that enable us to build the missing
NightHawk benchmark image resulting in a raised exception.
"""
# create a valid configuration with a missing NightHawk container image
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
images = generate_test_objects.generate_images(job_control)
images.nighthawk_binary_image = ""
# Generate a default Envoy source object.
generate_test_objects.generate_envoy_source(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark raise an exception from validate()
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build NightHawk image"
def test_execute_benchmark_missing_nighthawk_benchmark_image():
"""Validate an exception is raised if we cannot build the unspecified
NightHawk benchmark image.
"""
# create a valid configuration with a missing both NightHawk container images
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
images = generate_test_objects.generate_images(job_control)
images.nighthawk_benchmark_image = ""
# Generate a default Envoy source object
generate_test_objects.generate_envoy_source(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark shoud not throw an exception
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build NightHawk image"
def test_no_envoy_image_no_sources():
"""Test benchmark validation logic.
No Envoy image is specified, we expect the validation logic to
throw an exception since no sources are present
"""
# create a valid configuration with a missing Envoy image
job_control = _generate_default_job_control()
images = _generate_images(job_control)
images.envoy_image = ""
_generate_environment(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark shoud not throw an exception
with pytest.raises(full_docker.FullyDockerizedBenchmarkError) \
as validation_error:
benchmark.execute_benchmark()
assert str(validation_error.value) == "No source configuration specified"
def test_execute_benchmark_missing_envoy_source():
"""Validate that although sources are defined for NightHawk we raise an
exception due to the inability to build Envoy.
"""
# create a configuration with a missing Envoy image
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
images = generate_test_objects.generate_images(job_control)
images.envoy_image = ""
# Change the source identity to NightHawk
envoy_source = generate_test_objects.generate_envoy_source(job_control)
envoy_source.identity = envoy_source.SRCID_NIGHTHAWK
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark should raise an exception from validate()
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build Envoy image"
def test_source_to_build_envoy():
"""Validate we can build images from source.
Validate that sources are defined that enable us to build the Envoy image
We do not expect the validation logic to throw an exception
"""
# create a valid configuration with a missing Envoy image
job_control = _generate_default_job_control()
images = _generate_images(job_control)
images.envoy_image = ""
_generate_envoy_source(job_control)
env = job_control.environment
env.test_version = env.IPV_V4ONLY
env.envoy_path = 'envoy'
env.output_dir = 'some_output_dir'
env.test_dir = 'some_test_dir'
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
run_parameters = docker_image.DockerRunParameters(
command=['./benchmarks', '--log-cli-level=info', '-vvvv'],
environment=mock.ANY,
volumes=mock.ANY,
network_mode='host',
tty=True
)
# Mock the container invocation so that we don't try to pull an image
with mock.patch('src.lib.benchmark.base_benchmark.BaseBenchmark.run_image') \
as docker_mock:
benchmark.execute_benchmark()
docker_mock.assert_called_once_with(
'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag',
run_parameters)
