def test_execute_benchmark_with_envoy_source(mock_run_image): """Validate that if sources are defined that enable us to build the Envoy image we do not throw an exception. """ mock_run_image.return_value = b'benchmark_http_client output' # create a valid configuration with a missing Envoy image job_control = generate_test_objects.generate_default_job_control() images = generate_test_objects.generate_images(job_control) images.envoy_image = "" generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_environment(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") run_parameters = docker_image.DockerRunParameters( command=['./benchmarks', '--log-cli-level=info', '-vvvv'], environment=mock.ANY, volumes=mock.ANY, network_mode='host', tty=True) benchmark.execute_benchmark() mock_run_image.assert_called_once_with( 'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag', run_parameters)
def test_execute_with_building_envoy_images(mock_hashes_for_benchmarks, mock_have_build_options, mock_pull_image, mock_build_envoy, mock_build_nighthawk_binary, mock_build_nighthawk_benchmark, mock_execute, mock_symlink): """Verify that we invoke the build methods if we are not able to pull the required images for a benchmark """ # Build a default job control object with images job_control = generate_test_objects.generate_default_job_control() generate_test_objects.generate_images(job_control) generate_test_objects.generate_envoy_source(job_control) # mock_build_envoy.return_value = None mock_pull_image.side_effect = raise_docker_pull_exception mock_have_build_options.return_value = False mock_hashes_for_benchmarks.return_value = {'tag1', 'tag2'} # Instantiate the BenchmarkRunner so that it prepares the job control # objects for each benchmark benchmark = run_benchmark.BenchmarkRunner(job_control) benchmark.execute() mock_build_nighthawk_benchmark.assert_called() mock_build_nighthawk_binary.assert_called() mock_build_envoy.assert_called() mock_pull_image.assert_called() mock_symlink.assert_called() mock_execute.assert_has_calls([mock.call(), mock.call()])
def test_source_to_build_binaries(mock_cmd, mock_envoy_build, mock_nh_bin_build, mock_nh_bench_build): """Validate we can build binaries from source. Validate that sources are defined that enable us to build Envoy/Nighthawk We do not expect the validation logic to throw an exception """ # create a valid configuration with a missing Envoy image job_control = generate_test_objects.generate_default_job_control() generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_nighthawk_source(job_control) generate_test_objects.generate_environment(job_control) # Setup mock values mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static" mock_envoy_build.return_value = mock_envoy_path benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark") benchmark.execute_benchmark() assert benchmark._envoy_binary_path == mock_envoy_path mock_envoy_build.assert_called_once() mock_nh_bench_build.assert_called_once() mock_nh_bin_build.assert_called_once()
def test_nh_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build, mock_nh_bench_build): """Validate that an exception is raised which halts the benchmark execution when the Nighthawk build fails We expect an unhandled exception to surface from _prepare_nighthawk """ # Setup mock values mock_nh_bench_build.side_effect = subprocess.CalledProcessError(1, "bar") mock_envoy_path = "/home/ubuntu/envoy/bazel-bin/source/exe/envoy-static" mock_envoy_build.return_value = mock_envoy_path job_control = generate_test_objects.generate_default_job_control() generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_nighthawk_source(job_control) generate_test_objects.generate_environment(job_control) benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark") with pytest.raises(Exception) as build_exception: benchmark.execute_benchmark() assert str(build_exception.value ) == "Command 'bar' returned non-zero exit status 1." assert not benchmark._envoy_binary_path # We expect the nighthawk binary build to occur # before the benchmark build fails mock_nh_bin_build.assert_called_once() # Raising an exception during the nighthawk build should prevent control flow # from proceeding to build Envoy mock_envoy_build.assert_not_called()
def test_envoy_build_failure(mock_cmd, mock_envoy_build, mock_nh_bin_build, mock_nh_bench_build): """Validate that an exception is raised which halts the benchmark execution when the Envoy build fails We expect an unhandled exception to surface from _prepare_envoy """ # Setup mock values mock_envoy_build.side_effect = subprocess.CalledProcessError(1, "foo") job_control = generate_test_objects.generate_default_job_control() generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_nighthawk_source(job_control) generate_test_objects.generate_environment(job_control) benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark") with pytest.raises(Exception) as build_exception: benchmark.execute_benchmark() assert str(build_exception.value ) == "Command 'foo' returned non-zero exit status 1." assert not benchmark._envoy_binary_path # We expect the nighthawk build to occur before the Envoy build fails mock_nh_bench_build.assert_called_once() mock_nh_bin_build.assert_called_once()
def test_binary_benchmark_setup(mock_have_build_options, mock_symlink): """Verify that the unique methods to the binary benchmark workflow are in order""" job_control = proto_control.JobControl(remote=False, binary_benchmark=True) generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_nighthawk_source(job_control) benchmark = run_benchmark.BenchmarkRunner(job_control) mock_symlink.assert_called_with( 'source_url__hash_doesnt_really_matter_here__master', 'source_url__hash_doesnt_really_matter_here__master')
def test_execute_benchmark_envoy_source_only(): """Verify that we detect missing NightHawk sources """ job_control = generate_test_objects.generate_default_job_control() generate_test_objects.generate_envoy_source(job_control) benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging') with pytest.raises(base_benchmark.BenchmarkError) as benchmark_error: benchmark.execute_benchmark() assert str(benchmark_error.value) == \ "No source specified to build NightHawk image"
def test_execute_benchmark_missing_nighthawk_binary_image(): """Validate that no sources are defined that enable us to build the missing NightHawk benchmark image resulting in a raised exception. """ # create a valid configuration with a missing NightHawk container image job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) images = generate_test_objects.generate_images(job_control) images.nighthawk_binary_image = "" # Generate a default Envoy source object. generate_test_objects.generate_envoy_source(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark raise an exception from validate() with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build NightHawk image"
def test_execute_benchmark_no_environment(mock_benchmarks, mock_get_source_tree): """Verify that we fail a benchmark if no environment is set """ job_control = generate_test_objects.generate_default_job_control() # Add nighthawk and envoy sources generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_nighthawk_source(job_control) benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging') with pytest.raises(base_benchmark.BenchmarkEnvironmentError) as \ benchmark_error: benchmark.execute_benchmark() assert str(benchmark_error.value) == \ "No IP version is specified for the benchmark" mock_benchmarks.assert_called() mock_get_source_tree.assert_called()
def test_execute_benchmark_missing_nighthawk_benchmark_image(): """Validate an exception is raised if we cannot build the unspecified NightHawk benchmark image. """ # create a valid configuration with a missing both NightHawk container images job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) images = generate_test_objects.generate_images(job_control) images.nighthawk_benchmark_image = "" # Generate a default Envoy source object generate_test_objects.generate_envoy_source(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark shoud not throw an exception with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build NightHawk image"
def test_no_source_to_build_nh(): """Validate that we fail the entire process in the absence of NH sources Validate that even if Envoy sources are specified, the absence of Nighthawk sources will cause the program to fail. We expect the validation logic to throw an exception """ # create a valid configuration with a missing NightHawk container image job_control = proto_control.JobControl(remote=False, binary_benchmark=True) generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_environment(job_control) benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark should throw an exception with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build Nighthawk"
def test_execute_benchmark(mock_benchmarks, mock_get_source_tree, mock_run_command): """Verify that we fail a benchmark if no environment is set """ job_control = generate_test_objects.generate_default_job_control() # Add nighthawk and envoy sources generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_nighthawk_source(job_control) generate_test_objects.generate_environment(job_control) calls = [ mock.call( "bazel-bin/benchmarks/benchmarks " "--log-cli-level=info -vvvv -k test_http_h1_small " "benchmarks/", mock.ANY) ] benchmark = scavenging_benchmark.Benchmark(job_control, 'scavenging') benchmark.execute_benchmark() mock_benchmarks.assert_called() mock_get_source_tree.assert_called() mock_run_command.assert_has_calls(calls)
def test_execute_benchmark_missing_envoy_source(): """Validate that although sources are defined for NightHawk we raise an exception due to the inability to build Envoy. """ # create a configuration with a missing Envoy image job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) images = generate_test_objects.generate_images(job_control) images.envoy_image = "" # Change the source identity to NightHawk envoy_source = generate_test_objects.generate_envoy_source(job_control) envoy_source.identity = envoy_source.SRCID_NIGHTHAWK benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark should raise an exception from validate() with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build Envoy image"