def test_determine_envoy_hashes_from_source_pull_fail(
mock_copy_source_directory, mock_source_tree_pull):
"""
Verify that an exception is raised when we cannot determine Envoy hashes
from a job control object
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Setup mocks to simulate a source retrieval failure
mock_copy_source_directory.return_value = False
mock_source_tree_pull.return_value = False
manager = source_manager.SourceManager(job_control)
hashes = None
with pytest.raises(source_manager.SourceManagerError) as source_error:
hashes = manager.determine_envoy_hashes_from_source()
assert not hashes
assert str(source_error.value) == \
"Unable to obtain the source to determine commit hashes"
def test_get_image_hashes_from_disk_source(mock_run_command):
"""
Verify that we can determine Envoy hashes from source locations
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
source_repo = manager.get_source_repository(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
envoy_source_tree = source_tree.SourceTree(source_repo)
expected_hashes = {
'expected_previous_commit_hash', 'expected_baseline_hash'
}
origin = envoy_source_tree.get_origin()
assert origin
previous_hash = manager.get_image_hashes_from_disk_source(
envoy_source_tree, source_repo.commit_hash)
assert previous_hash == expected_hashes
def test_get_envoy_hashes_for_benchmark_additional_hashes(
mock_copy_source_directory, mock_run_command):
"""Verify that we can determine the hashes for the baseline and previous
Envoy Image.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Add an envoy image and specify additional versions to test
job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0"
for index in range(1, 4):
job_control.images.additional_envoy_images.append(
"envoyproxy/envoy:tag{i}".format(i=index))
# Setup mocks
mock_copy_source_directory.return_value = True
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
hashes = manager.get_envoy_hashes_for_benchmark()
# Since both a source and image was specified, we benchmark
# the source at its current and previous commit, as well as
# the other specified image tags.
expected_hashes = {
'tag1', 'tag2', 'tag3', 'v1.16.0', 'expected_previous_commit_hash',
'expected_baseline_hash'
}
assert hashes == expected_hashes
def test_get_image_hashes_from_disk_source_fail2(
mock_get_previous_commit_hash):
"""Verify that we raise an exception if we are not able to determine the
prior hash to a specified commit."""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Setup mocks
mock_get_previous_commit_hash.side_effect = _raise_source_tree_error
manager = source_manager.SourceManager(job_control)
tree = manager.get_source_tree(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
hashes = {}
with pytest.raises(source_manager.SourceManagerError) as source_error:
hashes = manager.get_image_hashes_from_disk_source(
tree, 'expected_baseline_hash')
assert not hashes
assert str(source_error.value) == \
"Unable to find a commit hash prior to [expected_baseline_hash]"
def test_determine_envoy_hashes_from_source2(mock_source_tree_pull,
mock_run_command):
"""
Verify that we can determine Envoy hashes from a source repository
This test exercises the else case where we use the head hash instead
of a specific envoy tag.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Add an envoy image and specify additional versions to test
job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0"
# Setup mocks
mock_source_tree_pull.return_value = True
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
hashes = manager.determine_envoy_hashes_from_source()
expected_hashes = {'v1.15.2', 'v1.16.0'}
assert hashes == expected_hashes
def _generate_default_source_manager():
"""Build a default SourceRepository object."""
control = proto_control.JobControl(remote=False, scavenging_benchmark=True)
control.source.add(
identity=proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY,
source_path='/some_random_envoy_directory',
commit_hash='v1.16.0')
return source_manager.SourceManager(control)
def _generate_default_source_manager():
"""Build a default SourceRepository object."""
control = proto_control.JobControl(remote=False, scavenging_benchmark=True)
control.source.add(
identity=proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK,
source_path='/where_nighthawk_code_lives',
)
return source_manager.SourceManager(control)
def test_find_all_images_from_specified_tags_build_envoy():
"""Verify that return no hashes and if we have to build Envoy"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
manager = source_manager.SourceManager(job_control)
tags = manager.find_all_images_from_specified_tags()
# Since the envoy image is not specified, we have not tags for a datum
expected_tags = set()
assert tags == expected_tags
def generate_image_manager_with_source_url():
"""Generate a source manager with a job control specifying remote repos
for images.
"""
job_control = proto_control.JobControl()
job_control.source.add(
identity=proto_source.SourceRepository.SRCID_ENVOY,
source_url='https://www.github.com/_some_random_repo_')
job_control.source.add(
identity=proto_source.SourceRepository.SRCID_NIGHTHAWK,
source_url='https://www.github.com/_nighthawk_repo_')
return source_manager.SourceManager(job_control)
def test_find_all_images_from_specified_tags_fail():
"""Verify that we raise an exception if no images are defined for any benchmarks"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
manager = source_manager.SourceManager(job_control)
hashes = []
with pytest.raises(source_manager.SourceManagerError) as source_error:
hashes = manager.find_all_images_from_specified_tags()
assert not hashes
assert str(source_error.value) == \
"No images are specified in the control document"
def test_validate_must_be_overidden():
"""Verify that the base _validate method must be overridden and raises
an exception otherwise.
"""
control = proto_control.JobControl(remote=False, scavenging_benchmark=True)
control.source.add(
identity=proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY,
source_path='/some_random_envoy_directory',
)
manager = source_manager.SourceManager(control)
builder = DerivedBuilder(manager)
with pytest.raises(NotImplementedError) as not_implemented:
builder.do_something()
assert str(not_implemented.value) == "Method should be overridden"
def test_get_source_tree_fail():
"""Verify that we raise an assertion if we are not able to find a
source repository.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
manager = source_manager.SourceManager(job_control)
tree = None
source_id = proto_source.SourceRepository.SourceIdentity.SRCID_UNSPECIFIED
with pytest.raises(source_manager.SourceManagerError) as source_error:
tree = manager.get_source_tree(source_id)
assert not tree
assert str(source_error.value) == \
"No Source tree defined for: SRCID_UNSPECIFIED"
def __init__(self, job_control: proto_control.JobControl,
benchmark_name: str) -> None:
""" Initializes the benchmark class
Args:
job_control: The protobuf object containing the parameters and locations
of benchmark artifacts
benchmark_name: The name of the benchmark to execute
"""
super(Benchmark, self).__init__(job_control, benchmark_name)
self._benchmark_dir = None
self._envoy_binary_path = job_control.environment.variables[
'ENVOY_PATH']
self._envoy_builder = None
self._nighthawk_builder = None
self._source_manager = source_manager.SourceManager(job_control)
def test_get_envoy_hashes_for_benchmark_minimal(mock_run_command):
"""Verify that we can determine the current and previous image
tags from a minimal job control object.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
image_config = _generate_default_benchmark_images(job_control)
image_config.envoy_image = "envoyproxy/envoy-dev:latest"
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
hashes = manager.get_envoy_hashes_for_benchmark()
assert hashes == {'mocked_hash_the_sequel', 'latest'}
def test_find_all_images_from_specified_tags_using_source(
mock_determine_envoy_hashes_from_source):
"""Verify that return no hashes and if we have to build Envoy"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
# Add an envoy image for us to use as a datum
job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0"
expected_tags = ['v1.15.2', 'v1.16.0']
mock_determine_envoy_hashes_from_source.return_value = expected_tags
manager = source_manager.SourceManager(job_control)
tags = manager.find_all_images_from_specified_tags()
assert tags == expected_tags
def test_get_source_tree():
"""Verify that we can return a source otree object. If no sources
are specified, we use the known default location from which to get
the source code.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
manager = source_manager.SourceManager(job_control)
for source_id in [
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY,
proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK
]:
tree = manager.get_source_tree(source_id)
assert tree.get_identity() == source_id
def test_get_build_options():
"""Verify that we can retrieve specified build options"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_envoy_source(job_control)
envoy_source = job_control.source[0]
expected_options = ["-c opt", "--jobs 4"]
for option in expected_options:
envoy_source.bazel_options.add(parameter=option)
manager = source_manager.SourceManager(job_control)
bazel_options = manager.get_build_options(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
assert bazel_options
assert all(
[option.parameter in expected_options for option in bazel_options])
def __init__(self, control: proto_control.JobControl) -> None:
"""Initialize the benchmark object.
Perform class member initialization and instantiate the underlying
object actually performing the test.
Args:
control: The Job Control object dictating the parameters governing the
benchmark
Returns:
None
"""
self._control = control
self._source_manager = source_manager.SourceManager(self._control)
self._test = []
self._setup_test()
def test_get_build_options_failure():
"""Verify that we raise an exception if no options are present in a source
repository.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_envoy_source(job_control)
manager = source_manager.SourceManager(job_control)
bazel_options = None
with pytest.raises(source_manager.SourceManagerError) as source_error:
bazel_options = manager.get_build_options(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
assert not bazel_options
assert str(source_error.value) == \
"No Bazel Options are defined in source: SRCID_ENVOY"
def _prepare_nighthawk(self) -> None:
"""Prepare the nighthawk source for the benchmark.
Checks out the nighthawk source if necessary, builds the client
and server binaries
"""
manager = source_manager.SourceManager(self._control)
# This builder needs to be a self object so that the temporary cache
# directory is not prematurely cleaned up
self._nighthawk_builder = nighthawk_builder.NightHawkBuilder(manager)
# FIXME: We build this for each envoy image that we test.
self._nighthawk_builder.build_nighthawk_benchmarks()
nighthawk_source = manager.get_source_tree(
proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK)
self._benchmark_dir = nighthawk_source.get_source_directory()
log.debug(f"NightHawk benchmark dir {self._benchmark_dir}")
def test_find_all_images_from_specified_sources(mock_copy_source_directory,
mock_run_command):
"""Verify that we can deterimine the previous commit hash from a source tree.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Setup mocks
mock_copy_source_directory.return_value = True
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
hashes = manager.find_all_images_from_specified_sources()
expected_hashes = {
'expected_previous_commit_hash', 'expected_baseline_hash'
}
assert hashes == expected_hashes
def test_determine_envoy_hashes_from_source(mock_copy_source_directory,
mock_run_command):
"""
Verify that we can determine Envoy hashes from a source repository
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Setup mocks
mock_copy_source_directory.return_value = True
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
hashes = manager.determine_envoy_hashes_from_source()
expected_hashes = {'mocked_hash_the_sequel', 'mocked_hash'}
assert hashes == expected_hashes
def test_find_all_images_from_specified_tags():
"""Verify that we can parse an image tag and deterimine the previous
image tag.
"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
# Add an envoy image and specify additional versions to test
job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0"
for index in range(1, 4):
job_control.images.additional_envoy_images.append(
"envoyproxy/envoy:tag{i}".format(i=index))
manager = source_manager.SourceManager(job_control)
tags = manager.find_all_images_from_specified_tags()
# We test all extra tags first and the specified image tags is always last
expected_tags = {'tag1', 'tag2', 'tag3', 'v1.16.0'}
assert tags == expected_tags
def test_have_build_options():
"""Verify that we can determine if build options exist"""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_envoy_source(job_control)
envoy_source = job_control.source[0]
expected_options = ["-c opt", "--jobs 4"]
for option in expected_options:
envoy_source.bazel_options.add(parameter=option)
# We construct build options for Envoy. NightHawk sources
# will not have any options specified
manager = source_manager.SourceManager(job_control)
bazel_options = manager.have_build_options(
proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK)
assert not bazel_options
bazel_options = manager.have_build_options(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
assert bazel_options
def test_get_image_hashes_from_disk_source(mock_copy_source_directory,
mock_run_command):
"""Verify that we can determine previous hash for a specified commit."""
job_control = proto_control.JobControl(remote=False,
scavenging_benchmark=True)
_generate_default_benchmark_images(job_control)
_generate_default_envoy_source(job_control)
# Setup mocks
mock_copy_source_directory.return_value = True
mock_run_command.side_effect = _run_command_side_effect
manager = source_manager.SourceManager(job_control)
tree = manager.get_source_tree(
proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY)
hashes = manager.get_image_hashes_from_disk_source(
tree, 'expected_baseline_hash')
expected_hashes = {
'expected_previous_commit_hash', 'expected_baseline_hash'
}
assert hashes == expected_hashes
