def test_execute_dockerized_benchmark_using_images_only(
mock_hashes_for_benchmarks, mock_have_build_options, mock_pull_image,
mock_execute, mock_run_image, mock_symlink):
"""Verify that we attempt to pull images if no sources are specified."""
# Build a default job control object with images
job_control = proto_control.JobControl(remote=False,
dockerized_benchmark=True)
generate_test_objects.generate_environment(job_control)
generate_test_objects.generate_images(job_control)
mock_run_image.return_value = b"benchmark_http_client output...."
mock_execute.return_value = None
mock_have_build_options.return_value = False
mock_hashes_for_benchmarks.return_value = {'tag1', 'tag2'}
# Instantiate the BenchmarkRunner so that it prepares the job control
# objects for each benchmark
benchmark = run_benchmark.BenchmarkRunner(job_control)
benchmark.execute()
mock_have_build_options.assert_called()
mock_pull_image.assert_called()
mock_symlink.assert_called()
mock_execute.assert_has_calls([mock.call(), mock.call()])
def test_execute_with_building_envoy_images(mock_hashes_for_benchmarks,
mock_have_build_options,
mock_pull_image, mock_build_envoy,
mock_build_nighthawk_binary,
mock_build_nighthawk_benchmark,
mock_execute, mock_symlink):
"""Verify that we invoke the build methods if we are not able to pull
the required images for a benchmark
"""
# Build a default job control object with images
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_images(job_control)
generate_test_objects.generate_envoy_source(job_control)
# mock_build_envoy.return_value = None
mock_pull_image.side_effect = raise_docker_pull_exception
mock_have_build_options.return_value = False
mock_hashes_for_benchmarks.return_value = {'tag1', 'tag2'}
# Instantiate the BenchmarkRunner so that it prepares the job control
# objects for each benchmark
benchmark = run_benchmark.BenchmarkRunner(job_control)
benchmark.execute()
mock_build_nighthawk_benchmark.assert_called()
mock_build_nighthawk_binary.assert_called()
mock_build_envoy.assert_called()
mock_pull_image.assert_called()
mock_symlink.assert_called()
mock_execute.assert_has_calls([mock.call(), mock.call()])
def main() -> int:
"""Driver module for benchmark.
This is the main function for starting a benchmark. It verifies that a
Job Control object was specified and from that starts the execution.
The benchmark object encapsulates the different benchmark modes and it
is responsible for selecting the correct classes to instantiate
"""
args = setup_options()
setup_logging()
if not args.jobcontrol:
print("No job control document specified. Use \"--help\" for usage")
return 1
job_control = load_control_doc(args.jobcontrol)
if job_control is None:
log.error(f"Unable to load or parse job control: {args.jobcontrol}")
return 1
bar = '=' * 20
log.debug(f"Job definition:\n{bar}\n{job_control}\n{bar}\n")
# Execute the benchmark given the contents of the job control file
benchmark = run_benchmark.BenchmarkRunner(job_control)
benchmark.execute()
return 0
def test_binary_benchmark_setup(mock_have_build_options, mock_symlink):
"""Verify that the unique methods to the binary benchmark workflow are in order"""
job_control = proto_control.JobControl(remote=False, binary_benchmark=True)
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
benchmark = run_benchmark.BenchmarkRunner(job_control)
mock_symlink.assert_called_with(
'source_url__hash_doesnt_really_matter_here__master',
'source_url__hash_doesnt_really_matter_here__master')
def test_benchmark_failure_if_no_benchmark_selected():
"""Verify that we raise an exception if no benchmark is configured to run.
"""
# Build a default job control object no benchmark selected
job_control = proto_control.JobControl(remote=False)
# Instantiate the BenchmarkRunner so that it prepares the job control
# objects for each benchmark
with pytest.raises(NotImplementedError) as not_implemented:
_ = run_benchmark.BenchmarkRunner(job_control)
assert str(not_implemented.value) == \
"No [Unspecified Benchmark] defined"
def test_execute_using_images_only(mock_hashes_for_benchmarks,
mock_have_build_options, mock_pull_image,
mock_execute, mock_symlink):
"""Verify that we attempt to pull images if no sources are specified."""
# Build a default job control object with images
job_control = generate_test_objects.generate_default_job_control()
generate_test_objects.generate_images(job_control)
mock_execute.return_value = None
mock_have_build_options.return_value = False
mock_hashes_for_benchmarks.return_value = {'tag1', 'tag2'}
# Instantiate the BenchmarkRunner so that it prepares the job control
# objects for each benchmark
benchmark = run_benchmark.BenchmarkRunner(job_control)
benchmark.execute()
mock_have_build_options.assert_called()
mock_pull_image.assert_called()
mock_symlink.assert_called()
mock_execute.assert_has_calls([mock.call(), mock.call()])