def test_execute_benchmark_with_envoy_source(mock_run_image):
"""Validate that if sources are defined that enable us to build the Envoy
image we do not throw an exception.
"""
mock_run_image.return_value = b'benchmark_http_client output'
# create a valid configuration with a missing Envoy image
job_control = generate_test_objects.generate_default_job_control()
images = generate_test_objects.generate_images(job_control)
images.envoy_image = ""
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
run_parameters = docker_image.DockerRunParameters(
command=['./benchmarks', '--log-cli-level=info', '-vvvv'],
environment=mock.ANY,
volumes=mock.ANY,
network_mode='host',
tty=True)
benchmark.execute_benchmark()
mock_run_image.assert_called_once_with(
'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag',
run_parameters)
def test_execute_with_building_envoy_images(mock_hashes_for_benchmarks,
mock_have_build_options,
mock_pull_image, mock_build_envoy,
mock_build_nighthawk_binary,
mock_build_nighthawk_benchmark,
mock_execute, mock_symlink):
"""Verify that we invoke the build methods if we are not able to pull
the required images for a benchmark
"""
# Build a default job control object with images
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_images(job_control)
generate_test_objects.generate_envoy_source(job_control)
# mock_build_envoy.return_value = None
mock_pull_image.side_effect = raise_docker_pull_exception
mock_have_build_options.return_value = False
mock_hashes_for_benchmarks.return_value = {'tag1', 'tag2'}
# Instantiate the BenchmarkRunner so that it prepares the job control
# objects for each benchmark
benchmark = run_benchmark.BenchmarkRunner(job_control)
benchmark.execute()
mock_build_nighthawk_benchmark.assert_called()
mock_build_nighthawk_binary.assert_called()
mock_build_envoy.assert_called()
mock_pull_image.assert_called()
mock_symlink.assert_called()
mock_execute.assert_has_calls([mock.call(), mock.call()])
def test_source_to_build_binaries(mock_cmd, mock_envoy_build,
mock_nh_bin_build, mock_nh_bench_build):
"""Validate we can build binaries from source.
Validate that sources are defined that enable us to build Envoy/Nighthawk
We do not expect the validation logic to throw an exception
"""
# create a valid configuration with a missing Envoy image
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
# Setup mock values
mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static"
mock_envoy_build.return_value = mock_envoy_path
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
benchmark.execute_benchmark()
assert benchmark._envoy_binary_path == mock_envoy_path
mock_envoy_build.assert_called_once()
mock_nh_bench_build.assert_called_once()
mock_nh_bin_build.assert_called_once()
def test_nh_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build,
mock_nh_bench_build):
"""Validate that an exception is raised which halts the benchmark execution
when the Nighthawk build fails
We expect an unhandled exception to surface from _prepare_nighthawk
"""
# Setup mock values
mock_nh_bench_build.side_effect = subprocess.CalledProcessError(1, "bar")
mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static"
mock_envoy_build.return_value = mock_envoy_path
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as build_exception:
benchmark.execute_benchmark()
assert str(build_exception.value
) == "Command 'bar' returned non-zero exit status 1."
assert not benchmark._envoy_binary_path
# We expect the nighthawk binary build to occur
# before the benchmark build fails
mock_nh_bin_build.assert_called_once()
# Raising an exception during the nighthawk build should prevent control flow
# from proceeding to build Envoy
mock_envoy_build.assert_not_called()
def test_envoy_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build,
mock_nh_bench_build):
"""Validate that an exception is raised which halts the benchmark execution
when the Envoy build fails
We expect an unhandled exception to surface from _prepare_envoy
"""
# Setup mock values
mock_envoy_build.side_effect = subprocess.CalledProcessError(1, "foo")
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as build_exception:
benchmark.execute_benchmark()
assert str(build_exception.value
) == "Command 'foo' returned non-zero exit status 1."
assert not benchmark._envoy_binary_path
# We expect the nighthawk build to occur before the Envoy build fails
mock_nh_bench_build.assert_called_once()
mock_nh_bin_build.assert_called_once()
def test_binary_benchmark_setup(mock_have_build_options, mock_symlink):
"""Verify that the unique methods to the binary benchmark workflow are in order"""
job_control = proto_control.JobControl(remote=False, binary_benchmark=True)
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
benchmark = run_benchmark.BenchmarkRunner(job_control)
mock_symlink.assert_called_with(
'source_url__hash_doesnt_really_matter_here__master',
'source_url__hash_doesnt_really_matter_here__master')
def test_execute_benchmark_envoy_source_only():
"""Verify that we detect missing NightHawk sources """
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkError) as benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == \
"No source specified to build NightHawk image"
def test_execute_benchmark_missing_nighthawk_binary_image():
"""Validate that no sources are defined that enable us to build the missing
NightHawk benchmark image resulting in a raised exception.
"""
# create a valid configuration with a missing NightHawk container image
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
images = generate_test_objects.generate_images(job_control)
images.nighthawk_binary_image = ""
# Generate a default Envoy source object.
generate_test_objects.generate_envoy_source(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark raise an exception from validate()
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build NightHawk image"
def test_execute_benchmark_no_environment(mock_benchmarks,
mock_get_source_tree):
"""Verify that we fail a benchmark if no environment is set """
job_control = generate_test_objects.generate_default_job_control()
# Add nighthawk and envoy sources
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkEnvironmentError) as \
benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == \
"No IP version is specified for the benchmark"
mock_benchmarks.assert_called()
mock_get_source_tree.assert_called()
def test_execute_benchmark_missing_nighthawk_benchmark_image():
"""Validate an exception is raised if we cannot build the unspecified
NightHawk benchmark image.
"""
# create a valid configuration with a missing both NightHawk container images
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
images = generate_test_objects.generate_images(job_control)
images.nighthawk_benchmark_image = ""
# Generate a default Envoy source object
generate_test_objects.generate_envoy_source(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark shoud not throw an exception
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build NightHawk image"
def test_no_source_to_build_nh():
"""Validate that we fail the entire process in the absence of NH sources
Validate that even if Envoy sources are specified, the absence of Nighthawk
sources will cause the program to fail.
We expect the validation logic to throw an exception
"""
# create a valid configuration with a missing NightHawk container image
job_control = proto_control.JobControl(remote=False, binary_benchmark=True)
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark should throw an exception
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build Nighthawk"
def test_execute_benchmark(mock_benchmarks, mock_get_source_tree,
mock_run_command):
"""Verify that we fail a benchmark if no environment is set """
job_control = generate_test_objects.generate_default_job_control()
# Add nighthawk and envoy sources
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
calls = [
mock.call(
"bazel-bin/benchmarks/benchmarks "
"--log-cli-level=info -vvvv -k test_http_h1_small "
"benchmarks/", mock.ANY)
]
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
benchmark.execute_benchmark()
mock_benchmarks.assert_called()
mock_get_source_tree.assert_called()
mock_run_command.assert_has_calls(calls)
def test_execute_benchmark_missing_envoy_source():
"""Validate that although sources are defined for NightHawk we raise an
exception due to the inability to build Envoy.
"""
# create a configuration with a missing Envoy image
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
images = generate_test_objects.generate_images(job_control)
images.envoy_image = ""
# Change the source identity to NightHawk
envoy_source = generate_test_objects.generate_envoy_source(job_control)
envoy_source.identity = envoy_source.SRCID_NIGHTHAWK
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark should raise an exception from validate()
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build Envoy image"