def test_no_source_to_build_nh2():
"""Validate that we fail to build nighthawk without sources.
Validate that no sources are defined that enable us to build the missing
NightHawk binary image.
We expect the validation logic to throw an exception
"""
# create a valid configuration with a missing both NightHawk container images
job_control = proto_control.JobControl(
remote=False,
scavenging_benchmark=True
)
images = _generate_images(job_control)
images.nighthawk_binary_image = ""
images.nighthawk_benchmark_image = ""
# Generate a default Envoy source object. Values aren't really checked at
# this stage since we have a missing Envoy image, nighthawk source validation
# should fail.
_generate_envoy_source(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark shoud not throw an exception
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build unspecified NightHawk image"
def _generate_default_job_control() -> proto_control.JobControl:
"""Generate a default job control object used in tests."""
job_control = proto_control.JobControl(
remote=False,
scavenging_benchmark=True
)
return job_control
def test_no_source_to_build_envoy():
"""Validate that we fail to build images without sources.
Validate that no sources are present that enable us to build the missing
Envoy image
We expect the validation logic to throw an exception
"""
# create a configuration with a missing Envoy image
job_control = proto_control.JobControl(
remote=False,
scavenging_benchmark=True
)
images = _generate_images(job_control)
images.envoy_image = ""
# Denote that the soure is for nighthawk. Values aren't really checked at
# this stage since we have a missing Envoy image and a nighthawk source
# validation should fail.
envoy_source = _generate_envoy_source(job_control)
envoy_source.identity = envoy_source.SRCID_NIGHTHAWK
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark shoud not throw an exception
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build unspecified Envoy image"
def test_get_image_hashes_from_disk_source(mock_run_command):
"""
Verify that we can determine Envoy hashes from source locations
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
source_repo = manager.get_source_repository(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
envoy_source_tree = source_tree.SourceTree(source_repo)
expected_hashes = {
'expected_previous_commit_hash', 'expected_baseline_hash'
}
origin = envoy_source_tree.get_origin()
assert origin
previous_hash = manager.get_image_hashes_from_disk_source(
envoy_source_tree, source_repo.commit_hash)
assert previous_hash == expected_hashes
def _load_json_doc(filename: str) -> proto_control.JobControl:
"""Load a disk file as JSON.
This function reads the specified filename and parses the contents
as JSON.
Args:
filename: The file whose contents are to be read as JSON data
Returns:
A JobControl object populated with the contents from the
specified JSON file
"""
contents = None
log.debug(f"Opening JSON file {filename}")
try:
with open(filename, 'r') as json_doc:
contents = json_format.Parse(json_doc.read(), proto_control.JobControl())
except FileNotFoundError as file_not_found:
log.exception(f"Unable to load {filename}: {file_not_found}")
except json_format.Error as json_parse_error:
log.exception(
f"Unable to parse JSON contents {filename}: {json_parse_error}")
return contents
def test_get_envoy_hashes_for_benchmark_additional_hashes(
mock_copy_source_directory, mock_run_command):
"""Verify that we can determine the hashes for the baseline and previous
Envoy Image.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Add an envoy image and specify additional versions to test
job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0"
for index in range(1, 4):
job_control.images.additional_envoy_images.append(
"envoyproxy/envoy:tag{i}".format(i=index))
# Setup mocks
mock_copy_source_directory.return_value = True
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
hashes = manager.get_envoy_hashes_for_benchmark()
# Since both a source and image was specified, we benchmark
# the source at its current and previous commit, as well as
# the other specified image tags.
expected_hashes = {
'tag1', 'tag2', 'tag3', 'v1.16.0', 'expected_previous_commit_hash',
'expected_baseline_hash'
}
assert hashes == expected_hashes
def test_get_image_hashes_from_disk_source_fail2(
mock_get_previous_commit_hash):
"""Verify that we raise an exception if we are not able to determine the
prior hash to a specified commit."""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Setup mocks
mock_get_previous_commit_hash.side_effect = _raise_source_tree_error
manager = source_manager.SourceManager(job_control)
tree = manager.get_source_tree(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
hashes = {}
with pytest.raises(source_manager.SourceManagerError) as source_error:
hashes = manager.get_image_hashes_from_disk_source(
tree, 'expected_baseline_hash')
assert not hashes
assert str(source_error.value) == \
"Unable to find a commit hash prior to [expected_baseline_hash]"
def test_determine_envoy_hashes_from_source2(mock_source_tree_pull,
mock_run_command):
"""
Verify that we can determine Envoy hashes from a source repository
This test exercises the else case where we use the head hash instead
of a specific envoy tag.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Add an envoy image and specify additional versions to test
job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0"
# Setup mocks
mock_source_tree_pull.return_value = True
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
hashes = manager.determine_envoy_hashes_from_source()
expected_hashes = {'v1.15.2', 'v1.16.0'}
assert hashes == expected_hashes
def _load_yaml_doc(filename: str) -> proto_control.JobControl:
"""Load a disk file as YAML.
This function reads the specified filename and parses the contents
as YAML.
Args:
filename: The file whose contents are to be read as YAML data
Returns:
A JobControl object populated with the contents from the
specified YAML file
"""
log.debug(f"Opening YAML file {filename}")
contents = None
try:
with open(filename, 'r') as yaml_doc:
contents = yaml.load(yaml_doc.read())
contents = json_format.Parse(
json.dumps(contents), proto_control.JobControl())
except FileNotFoundError as file_not_found:
log.exception(f"Unable to load {filename}: {file_not_found}")
except json_format.Error as yaml_parse_error:
log.exception(
f"Unable to parse YAML contents {filename}: {yaml_parse_error}")
return contents
def test_determine_envoy_hashes_from_source_pull_fail(
mock_copy_source_directory, mock_source_tree_pull):
"""
Verify that an exception is raised when we cannot determine Envoy hashes
from a job control object
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Setup mocks to simulate a source retrieval failure
mock_copy_source_directory.return_value = False
mock_source_tree_pull.return_value = False
manager = source_manager.SourceManager(job_control)
hashes = None
with pytest.raises(source_manager.SourceManagerError) as source_error:
hashes = manager.determine_envoy_hashes_from_source()
assert not hashes
assert str(source_error.value) == \
"Unable to obtain the source to determine commit hashes"
def test_execute_dockerized_benchmark_using_images_only(
mock_hashes_for_benchmarks, mock_have_build_options, mock_pull_image,
mock_execute, mock_run_image, mock_symlink):
"""Verify that we attempt to pull images if no sources are specified."""
# Build a default job control object with images
job_control = proto_control.JobControl(remote=False,
dockerized_benchmark=True)
generate_test_objects.generate_environment(job_control)
generate_test_objects.generate_images(job_control)
mock_run_image.return_value = b"benchmark_http_client output...."
mock_execute.return_value = None
mock_have_build_options.return_value = False
mock_hashes_for_benchmarks.return_value = {'tag1', 'tag2'}
# Instantiate the BenchmarkRunner so that it prepares the job control
# objects for each benchmark
benchmark = run_benchmark.BenchmarkRunner(job_control)
benchmark.execute()
mock_have_build_options.assert_called()
mock_pull_image.assert_called()
mock_symlink.assert_called()
mock_execute.assert_has_calls([mock.call(), mock.call()])
def _create_new_job_control(self, envoy_image) -> proto_control.JobControl:
"""Duplicate the job control for a specific benchmark run.
This method creates a new job control object and sets the commit hash for
the envoy revision being tested
Args:
envoy_image: The envoy image name being tested. This is expected to be
in the format "envoyproxy/envoy-dev:tag".
Returns:
A job control document containing the hash and image name being tested
"""
image_hash = envoy_image.split(':')[-1]
output_dir = os.path.join(self._control.environment.output_dir,
image_hash)
new_job_control = proto_control.JobControl()
new_job_control.CopyFrom(self._control)
new_job_control.images.envoy_image = envoy_image
new_job_control.environment.output_dir = output_dir
self._create_symlink_for_test_artifacts(output_dir, image_hash)
return new_job_control
def _generate_default_source_manager():
"""Build a default SourceRepository object."""
control = proto_control.JobControl(remote=False, scavenging_benchmark=True)
control.source.add(
identity=proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK,
source_path='/where_nighthawk_code_lives',
)
return source_manager.SourceManager(control)
def _generate_default_source_manager():
"""Build a default SourceRepository object."""
control = proto_control.JobControl(remote=False, scavenging_benchmark=True)
control.source.add(
identity=proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY,
source_path='/some_random_envoy_directory',
commit_hash='v1.16.0')
return source_manager.SourceManager(control)
def test_binary_benchmark_setup(mock_have_build_options, mock_symlink):
"""Verify that the unique methods to the binary benchmark workflow are in order"""
job_control = proto_control.JobControl(remote=False, binary_benchmark=True)
generate_test_objects.generate_envoy_source(job_control)
generate_test_objects.generate_nighthawk_source(job_control)
benchmark = run_benchmark.BenchmarkRunner(job_control)
mock_symlink.assert_called_with(
'source_url__hash_doesnt_really_matter_here__master',
'source_url__hash_doesnt_really_matter_here__master')
def test_find_all_images_from_specified_tags_build_envoy():
"""Verify that return no hashes and if we have to build Envoy"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
manager = source_manager.SourceManager(job_control)
tags = manager.find_all_images_from_specified_tags()
# Since the envoy image is not specified, we have not tags for a datum
expected_tags = set()
assert tags == expected_tags
def generate_image_manager_with_source_url():
"""Generate a source manager with a job control specifying remote repos
for images.
"""
job_control = proto_control.JobControl()
job_control.source.add(
identity=proto_source.SourceRepository.SRCID_ENVOY,
source_url='https://www.github.com/_some_random_repo_')
job_control.source.add(
identity=proto_source.SourceRepository.SRCID_NIGHTHAWK,
source_url='https://www.github.com/_nighthawk_repo_')
return source_manager.SourceManager(job_control)
def test_benchmark_failure_if_no_benchmark_selected():
"""Verify that we raise an exception if no benchmark is configured to run.
"""
# Build a default job control object no benchmark selected
job_control = proto_control.JobControl(remote=False)
# Instantiate the BenchmarkRunner so that it prepares the job control
# objects for each benchmark
with pytest.raises(NotImplementedError) as not_implemented:
_ = run_benchmark.BenchmarkRunner(job_control)
assert str(not_implemented.value) == \
"No [Unspecified Benchmark] defined"
def test_find_all_images_from_specified_tags_fail():
"""Verify that we raise an exception if no images are defined for any benchmarks"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
manager = source_manager.SourceManager(job_control)
hashes = []
with pytest.raises(source_manager.SourceManagerError) as source_error:
hashes = manager.find_all_images_from_specified_tags()
assert not hashes
assert str(source_error.value) == \
"No images are specified in the control document"
def test_validate_must_be_overidden():
"""Verify that the base _validate method must be overridden and raises
an exception otherwise.
"""
control = proto_control.JobControl(remote=False, scavenging_benchmark=True)
control.source.add(
identity=proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY,
source_path='/some_random_envoy_directory',
)
manager = source_manager.SourceManager(control)
builder = DerivedBuilder(manager)
with pytest.raises(NotImplementedError) as not_implemented:
builder.do_something()
assert str(not_implemented.value) == "Method should be overridden"
def test_find_all_images_from_specified_tags_using_source(
mock_determine_envoy_hashes_from_source):
"""Verify that return no hashes and if we have to build Envoy"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
# Add an envoy image for us to use as a datum
job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0"
expected_tags = ['v1.15.2', 'v1.16.0']
mock_determine_envoy_hashes_from_source.return_value = expected_tags
manager = source_manager.SourceManager(job_control)
tags = manager.find_all_images_from_specified_tags()
assert tags == expected_tags
def test_get_envoy_hashes_for_benchmark_minimal(mock_run_command):
"""Verify that we can determine the current and previous image
tags from a minimal job control object.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
image_config = _generate_default_benchmark_images(job_control)
image_config.envoy_image = "envoyproxy/envoy-dev:latest"
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
hashes = manager.get_envoy_hashes_for_benchmark()
assert hashes == {'mocked_hash_the_sequel', 'latest'}
def test_get_source_tree():
"""Verify that we can return a source otree object. If no sources
are specified, we use the known default location from which to get
the source code.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
manager = source_manager.SourceManager(job_control)
for source_id in [
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY,
proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK
]:
tree = manager.get_source_tree(source_id)
assert tree.get_identity() == source_id
def test_get_source_tree_fail():
"""Verify that we raise an assertion if we are not able to find a
source repository.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
manager = source_manager.SourceManager(job_control)
tree = None
source_id = proto_source.SourceRepository.SourceIdentity.SRCID_UNSPECIFIED
with pytest.raises(source_manager.SourceManagerError) as source_error:
tree = manager.get_source_tree(source_id)
assert not tree
assert str(source_error.value) == \
"No Source tree defined for: SRCID_UNSPECIFIED"
def _create_new_source_job_control(self, nh_source, envoy_source, envoy_hash) \
-> proto_control.JobControl:
"""Duplicate the job control for a specific benchmark run.
This method creates a new job control object for a single binary benchmark
Args:
nh_source: the nighthawk source to run
envoy_source: the envoy source to test
Returns:
A job control document containing the Nighthawk and Envoy source specified
"""
# Generate a unique identifier for the individual run
# 1. Specifies whether source is provided via source_path or source_url
# 2. Specifies the primary commit hash or tag, if provided
# 3. Specifies the branch, if provided
# Ex: source_url__v1.16.0__branch_name
identifier = str(envoy_source.WhichOneof('source_location'))
if envoy_hash:
identifier += '__'
identifier += envoy_hash
if envoy_source.branch:
identifier += '__'
identifier += envoy_source.branch
# Create a new SourceRepository representing just one Envoy build target
new_envoy_source = proto_source.SourceRepository()
new_envoy_source.CopyFrom(envoy_source)
new_envoy_source.commit_hash = envoy_hash
del new_envoy_source.additional_hashes[:]
output_dir = os.path.join(self._control.environment.output_dir,
identifier)
new_job_control = proto_control.JobControl()
new_job_control.CopyFrom(self._control)
del new_job_control.source[:]
new_job_control.source.extend([nh_source, new_envoy_source])
new_job_control.environment.output_dir = output_dir
log.debug(f"Creating new Binary job for {new_envoy_source}")
log.debug(f"Job:\n{new_job_control}")
self._create_symlink_for_test_artifacts(output_dir, identifier)
return new_job_control
def test_get_build_options():
"""Verify that we can retrieve specified build options"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_envoy_source(job_control)
envoy_source = job_control.source[0]
expected_options = ["-c opt", "--jobs 4"]
for option in expected_options:
envoy_source.bazel_options.add(parameter=option)
manager = source_manager.SourceManager(job_control)
bazel_options = manager.get_build_options(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
assert bazel_options
assert all(
[option.parameter in expected_options for option in bazel_options])
def test_no_valid_envoy_binary(mock_nh_bin_build, mock_nh_bench_build):
"""Validate that we fail when Envoy sources are not present,
and no binary is specified.
We expect an unhandled exception to surface from _prepare_envoy()
"""
# create a valid configuration with a missing both NightHawk container images
job_control = proto_control.JobControl(remote=False, binary_benchmark=True)
job_control.environment.variables['ENVOY_PATH'] = '/dev/null/foo'
generate_test_objects.generate_nighthawk_source(job_control)
generate_test_objects.generate_environment(job_control)
benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark")
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"ENVOY_PATH environment variable specified, but invalid"
def test_get_build_options_failure():
"""Verify that we raise an exception if no options are present in a source
repository.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_envoy_source(job_control)
manager = source_manager.SourceManager(job_control)
bazel_options = None
with pytest.raises(source_manager.SourceManagerError) as source_error:
bazel_options = manager.get_build_options(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
assert not bazel_options
assert str(source_error.value) == \
"No Bazel Options are defined in source: SRCID_ENVOY"
def test_execute_benchmark_missing_nighthawk_binary_image():
"""Validate that no sources are defined that enable us to build the missing
NightHawk benchmark image resulting in a raised exception.
"""
# create a valid configuration with a missing NightHawk container image
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
images = generate_test_objects.generate_images(job_control)
images.nighthawk_binary_image = ""
# Generate a default Envoy source object.
generate_test_objects.generate_envoy_source(job_control)
benchmark = full_docker.Benchmark(job_control, "test_benchmark")
# Calling execute_benchmark raise an exception from validate()
with pytest.raises(Exception) as validation_exception:
benchmark.execute_benchmark()
assert str(validation_exception.value) == \
"No source specified to build NightHawk image"
def test_find_all_images_from_specified_sources(mock_copy_source_directory,
mock_run_command):
"""Verify that we can deterimine the previous commit hash from a source tree.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Setup mocks
mock_copy_source_directory.return_value = True
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
hashes = manager.find_all_images_from_specified_sources()
expected_hashes = {
'expected_previous_commit_hash', 'expected_baseline_hash'
}
assert hashes == expected_hashes
