def test_source_to_build_binaries(mock_cmd, mock_envoy_build,
mock_nh_bin_build, mock_nh_bench_build):
"""Validate we can build binaries from source.
Validate that sources are defined that enable us to build Envoy/Nighthawk
We do not expect the validation logic to throw an exception
"""
# create a valid configuration with a missing Envoy image
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
# Setup mock values
mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static"
mock_envoy_build.return_value = mock_envoy_path
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
benchmark.execute_benchmark()
assert benchmark._envoy_binary_path == mock_envoy_path
mock_envoy_build.assert_called_once()
mock_nh_bench_build.assert_called_once()
mock_nh_bin_build.assert_called_once()
def test_nh_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build,
mock_nh_bench_build):
"""Validate that an exception is raised which halts the benchmark execution
when the Nighthawk build fails
We expect an unhandled exception to surface from _prepare_nighthawk
"""
# Setup mock values
mock_nh_bench_build.side_effect = subprocess.CalledProcessError(1, "bar")
mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static"
mock_envoy_build.return_value = mock_envoy_path
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as build_exception:
benchmark.execute_benchmark()
assert str(build_exception.value
) == "Command 'bar' returned non-zero exit status 1."
assert not benchmark._envoy_binary_path
# We expect the nighthawk binary build to occur
# before the benchmark build fails
mock_nh_bin_build.assert_called_once()
# Raising an exception during the nighthawk build should prevent control flow
# from proceeding to build Envoy
mock_envoy_build.assert_not_called()
def test_envoy_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build,
mock_nh_bench_build):
"""Validate that an exception is raised which halts the benchmark execution
when the Envoy build fails
We expect an unhandled exception to surface from _prepare_envoy
"""
# Setup mock values
mock_envoy_build.side_effect = subprocess.CalledProcessError(1, "foo")
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as build_exception:
benchmark.execute_benchmark()
assert str(build_exception.value
) == "Command 'foo' returned non-zero exit status 1."
assert not benchmark._envoy_binary_path
# We expect the nighthawk build to occur before the Envoy build fails
mock_nh_bench_build.assert_called_once()
mock_nh_bin_build.assert_called_once()
def test_binary_benchmark_setup(mock_have_build_options, mock_symlink):
"""Verify that the unique methods to the binary benchmark workflow are in order"""
job_control = proto_control.JobControl(remote=False, binary_benchmark=True)
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
benchmark = run_benchmark.BenchmarkRunner(job_control)
mock_symlink.assert_called_with(
'source_url__hash_doesnt_really_matter_here__master',
'source_url__hash_doesnt_really_matter_here__master')
def test_execute_benchmark_nighthawk_source_only():
"""Verify that we detect missing Envoy sources """
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_nighthawk_source(job_control)
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkError) as benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == \
"No source specified to build Envoy image"
def test_no_valid_envoy_binary(mock_nh_bin_build, mock_nh_bench_build):
"""Validate that we fail when Envoy sources are not present,
and no binary is specified.
We expect an unhandled exception to surface from _prepare_envoy()
"""
# create a valid configuration with a missing both NightHawk container images
job_control = proto_control.JobControl(remote=False, binary_benchmark=True)
job_control.environment.variables['ENVOY_PATH'] = '/dev/null/foo'
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"ENVOY_PATH environment variable specified, but invalid"
def test_execute_benchmark_no_environment(mock_benchmarks,
mock_get_source_tree):
"""Verify that we fail a benchmark if no environment is set """
job_control = generate_test_objects.generate_default_job_control()
# Add nighthawk and envoy sources
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
with pytest.raises(base_benchmark.BenchmarkEnvironmentError) as \
benchmark_error:
benchmark.execute_benchmark()
assert str(benchmark_error.value) == \
"No IP version is specified for the benchmark"
mock_benchmarks.assert_called()
mock_get_source_tree.assert_called()
def test_execute_benchmark(mock_benchmarks, mock_get_source_tree,
mock_run_command):
"""Verify that we fail a benchmark if no environment is set """
job_control = generate_test_objects.generate_default_job_control()
# Add nighthawk and envoy sources
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
calls = [
mock.call(
"bazel-bin/benchmarks/benchmarks "
"--log-cli-level=info -vvvv -k test_http_h1_small "
"benchmarks/", mock.ANY)
]
benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging')
benchmark.execute_benchmark()
mock_benchmarks.assert_called()
mock_get_source_tree.assert_called()
mock_run_command.assert_has_calls(calls)