def test_execute_with_building_envoy_images(mock_hashes_for_benchmarks,
mock_have_build_options,
mock_pull_image, mock_build_envoy,
mock_build_nighthawk_binary,
mock_build_nighthawk_benchmark,
mock_execute, mock_symlink):
"""Verify that we invoke the build methods if we are not able to pull
the required images for a benchmark
"""
# Build a default job control object with images
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_images(job_control)
generate_test_objects.generate_envoy_source(job_control)
# mock_build_envoy.return_value = None
mock_pull_image.side_effect = raise_docker_pull_exception
mock_have_build_options.return_value = False
mock_hashes_for_benchmarks.return_value = {'tag1', 'tag2'}
# Instantiate the BenchmarkRunner so that it prepares the job control
# objects for each benchmark
benchmark = run_benchmark.BenchmarkRunner(job_control)
benchmark.execute()
mock_build_nighthawk_benchmark.assert_called()
mock_build_nighthawk_binary.assert_called()
mock_build_envoy.assert_called()
mock_pull_image.assert_called()
mock_symlink.assert_called()
mock_execute.assert_has_calls([mock.call(), mock.call()])
def test_execute_benchmark_with_envoy_source(mock_run_image):
"""Validate that if sources are defined that enable us to build the Envoy
image we do not throw an exception.
"""
mock_run_image.return_value = b'benchmark_http_client output'
# create a valid configuration with a missing Envoy image
job_control = generate_test_objects.generate_default_job_control()
images = generate_test_objects.generate_images(job_control)
images.envoy_image = ""
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
run_parameters = docker_image.DockerRunParameters(
command=['./benchmarks', '--log-cli-level=info', '-vvvv'],
environment=mock.ANY,
volumes=mock.ANY,
network_mode='host',
tty=True)
benchmark.execute_benchmark()
mock_run_image.assert_called_once_with(
'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag',
run_parameters)
def test_execute_benchmark_using_images_only(mock_run_image):
"""Validate we execute the benchmark with only images specified."""
mock_run_image.return_value = b'benchmark_http_client output'
# create a valid configuration defining images only for benchmark
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_images(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
run_parameters = docker_image.DockerRunParameters(
command=['./benchmarks', '--log-cli-level=info', '-vvvv'],
environment=mock.ANY,
volumes=mock.ANY,
network_mode='host',
tty=True)
# Calling execute_benchmark shoud not throw an exception
benchmark.execute_benchmark()
mock_run_image.assert_called_once_with(
'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag',
run_parameters)
def test_source_to_build_binaries(mock_cmd, mock_envoy_build,
mock_nh_bin_build, mock_nh_bench_build):
"""Validate we can build binaries from source.
Validate that sources are defined that enable us to build Envoy/Nighthawk
We do not expect the validation logic to throw an exception
"""
# create a valid configuration with a missing Envoy image
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
# Setup mock values
mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static"
mock_envoy_build.return_value = mock_envoy_path
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
benchmark.execute_benchmark()
assert benchmark._envoy_binary_path == mock_envoy_path
mock_envoy_build.assert_called_once()
mock_nh_bench_build.assert_called_once()
mock_nh_bin_build.assert_called_once()
def test_nh_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build,
mock_nh_bench_build):
"""Validate that an exception is raised which halts the benchmark execution
when the Nighthawk build fails
We expect an unhandled exception to surface from _prepare_nighthawk
"""
# Setup mock values
mock_nh_bench_build.side_effect = subprocess.CalledProcessError(1, "bar")
mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static"
mock_envoy_build.return_value = mock_envoy_path
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as build_exception:
benchmark.execute_benchmark()
assert str(build_exception.value
) == "Command 'bar' returned non-zero exit status 1."
assert not benchmark._envoy_binary_path
# We expect the nighthawk binary build to occur
# before the benchmark build fails
mock_nh_bin_build.assert_called_once()
# Raising an exception during the nighthawk build should prevent control flow
# from proceeding to build Envoy
mock_envoy_build.assert_not_called()
def test_envoy_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build,
mock_nh_bench_build):
"""Validate that an exception is raised which halts the benchmark execution
when the Envoy build fails
We expect an unhandled exception to surface from _prepare_envoy
"""
# Setup mock values
mock_envoy_build.side_effect = subprocess.CalledProcessError(1, "foo")
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as build_exception:
benchmark.execute_benchmark()
assert str(build_exception.value
) == "Command 'foo' returned non-zero exit status 1."
assert not benchmark._envoy_binary_path
# We expect the nighthawk build to occur before the Envoy build fails
mock_nh_bench_build.assert_called_once()
mock_nh_bin_build.assert_called_once()
def test_execute_benchmark_no_images_or_sources():
"""Verify the benchmark fails if no images or sources are present """
job_control = generate_test_objects.generate_default_job_control()
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkError) as benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == "No source configuration specified"
def test_execute_benchmark_envoy_source_only():
"""Verify that we detect missing NightHawk sources """
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkError) as benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == \
"No source specified to build NightHawk image"
def test_no_sources():
"""Test benchmark validation logic.
We expect the validation logic to throw an exception,
since no sources are present
"""
# create a valid configuration with no images
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark should throw an exception
with pytest.raises(binary_benchmark.BinaryBenchmarkError) \
as validation_error:
benchmark.execute_benchmark()
assert str(validation_error.value) == "No source configuration specified"
def test_execute_benchmark_no_image_or_sources():
"""Verify that the validation logic raises an exception since we are unable to
build a required Envoy image.
"""
# create a valid configuration with a missing Envoy image
job_control = generate_test_objects.generate_default_job_control()
images = generate_test_objects.generate_images(job_control)
images.envoy_image = ""
generate_test_objects.generate_environment(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark shoud not throw an exception
with pytest.raises(base_benchmark.BenchmarkError) \
as validation_error:
benchmark.execute_benchmark()
assert str(validation_error.value) == "No source configuration specified"
def test_execute_benchmark_no_environment(mock_benchmarks,
mock_get_source_tree):
"""Verify that we fail a benchmark if no environment is set """
job_control = generate_test_objects.generate_default_job_control()
# Add nighthawk and envoy sources
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkEnvironmentError) as \
benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == \
"No IP version is specified for the benchmark"
mock_benchmarks.assert_called()
mock_get_source_tree.assert_called()
def test_execute_using_images_only(mock_hashes_for_benchmarks,
mock_have_build_options, mock_pull_image,
mock_execute, mock_symlink):
"""Verify that we attempt to pull images if no sources are specified."""
# Build a default job control object with images
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_images(job_control)
mock_execute.return_value = None
mock_have_build_options.return_value = False
mock_hashes_for_benchmarks.return_value = {'tag1', 'tag2'}
# Instantiate the BenchmarkRunner so that it prepares the job control
# objects for each benchmark
benchmark = run_benchmark.BenchmarkRunner(job_control)
benchmark.execute()
mock_have_build_options.assert_called()
mock_pull_image.assert_called()
mock_symlink.assert_called()
mock_execute.assert_has_calls([mock.call(), mock.call()])
def test_execute_benchmark(mock_benchmarks, mock_get_source_tree,
mock_run_command):
"""Verify that we fail a benchmark if no environment is set """
job_control = generate_test_objects.generate_default_job_control()
# Add nighthawk and envoy sources
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
calls = [
mock.call(
"bazel-bin/benchmarks/benchmarks "
"--log-cli-level=info -vvvv -k test_http_h1_small "
"benchmarks/", mock.ANY)
]
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
benchmark.execute_benchmark()
mock_benchmarks.assert_called()
mock_get_source_tree.assert_called()
mock_run_command.assert_has_calls(calls)
