def test_determine_envoy_hashes_from_source_pull_fail( mock_copy_source_directory, mock_source_tree_pull): """ Verify that an exception is raised when we cannot determine Envoy hashes from a job control object """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Setup mocks to simulate a source retrieval failure mock_copy_source_directory.return_value = False mock_source_tree_pull.return_value = False manager = source_manager.SourceManager(job_control) hashes = None with pytest.raises(source_manager.SourceManagerError) as source_error: hashes = manager.determine_envoy_hashes_from_source() assert not hashes assert str(source_error.value) == \ "Unable to obtain the source to determine commit hashes"
def test_get_image_hashes_from_disk_source(mock_run_command): """ Verify that we can determine Envoy hashes from source locations """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) source_repo = manager.get_source_repository( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) envoy_source_tree = source_tree.SourceTree(source_repo) expected_hashes = { 'expected_previous_commit_hash', 'expected_baseline_hash' } origin = envoy_source_tree.get_origin() assert origin previous_hash = manager.get_image_hashes_from_disk_source( envoy_source_tree, source_repo.commit_hash) assert previous_hash == expected_hashes
def test_get_envoy_hashes_for_benchmark_additional_hashes( mock_copy_source_directory, mock_run_command): """Verify that we can determine the hashes for the baseline and previous Envoy Image. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Add an envoy image and specify additional versions to test job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0" for index in range(1, 4): job_control.images.additional_envoy_images.append( "envoyproxy/envoy:tag{i}".format(i=index)) # Setup mocks mock_copy_source_directory.return_value = True mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) hashes = manager.get_envoy_hashes_for_benchmark() # Since both a source and image was specified, we benchmark # the source at its current and previous commit, as well as # the other specified image tags. expected_hashes = { 'tag1', 'tag2', 'tag3', 'v1.16.0', 'expected_previous_commit_hash', 'expected_baseline_hash' } assert hashes == expected_hashes
def test_get_image_hashes_from_disk_source_fail2( mock_get_previous_commit_hash): """Verify that we raise an exception if we are not able to determine the prior hash to a specified commit.""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Setup mocks mock_get_previous_commit_hash.side_effect = _raise_source_tree_error manager = source_manager.SourceManager(job_control) tree = manager.get_source_tree( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) hashes = {} with pytest.raises(source_manager.SourceManagerError) as source_error: hashes = manager.get_image_hashes_from_disk_source( tree, 'expected_baseline_hash') assert not hashes assert str(source_error.value) == \ "Unable to find a commit hash prior to [expected_baseline_hash]"
def test_determine_envoy_hashes_from_source2(mock_source_tree_pull, mock_run_command): """ Verify that we can determine Envoy hashes from a source repository This test exercises the else case where we use the head hash instead of a specific envoy tag. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Add an envoy image and specify additional versions to test job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0" # Setup mocks mock_source_tree_pull.return_value = True mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) hashes = manager.determine_envoy_hashes_from_source() expected_hashes = {'v1.15.2', 'v1.16.0'} assert hashes == expected_hashes
def _generate_default_source_manager(): """Build a default SourceRepository object.""" control = proto_control.JobControl(remote=False, scavenging_benchmark=True) control.source.add( identity=proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY, source_path='/some_random_envoy_directory', commit_hash='v1.16.0') return source_manager.SourceManager(control)
def _generate_default_source_manager(): """Build a default SourceRepository object.""" control = proto_control.JobControl(remote=False, scavenging_benchmark=True) control.source.add( identity=proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK, source_path='/where_nighthawk_code_lives', ) return source_manager.SourceManager(control)
def test_find_all_images_from_specified_tags_build_envoy(): """Verify that return no hashes and if we have to build Envoy""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) manager = source_manager.SourceManager(job_control) tags = manager.find_all_images_from_specified_tags() # Since the envoy image is not specified, we have not tags for a datum expected_tags = set() assert tags == expected_tags
def generate_image_manager_with_source_url(): """Generate a source manager with a job control specifying remote repos for images. """ job_control = proto_control.JobControl() job_control.source.add( identity=proto_source.SourceRepository.SRCID_ENVOY, source_url='https://www.github.com/_some_random_repo_') job_control.source.add( identity=proto_source.SourceRepository.SRCID_NIGHTHAWK, source_url='https://www.github.com/_nighthawk_repo_') return source_manager.SourceManager(job_control)
def test_find_all_images_from_specified_tags_fail(): """Verify that we raise an exception if no images are defined for any benchmarks""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) manager = source_manager.SourceManager(job_control) hashes = [] with pytest.raises(source_manager.SourceManagerError) as source_error: hashes = manager.find_all_images_from_specified_tags() assert not hashes assert str(source_error.value) == \ "No images are specified in the control document"
def test_validate_must_be_overidden(): """Verify that the base _validate method must be overridden and raises an exception otherwise. """ control = proto_control.JobControl(remote=False, scavenging_benchmark=True) control.source.add( identity=proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY, source_path='/some_random_envoy_directory', ) manager = source_manager.SourceManager(control) builder = DerivedBuilder(manager) with pytest.raises(NotImplementedError) as not_implemented: builder.do_something() assert str(not_implemented.value) == "Method should be overridden"
def test_get_source_tree_fail(): """Verify that we raise an assertion if we are not able to find a source repository. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) manager = source_manager.SourceManager(job_control) tree = None source_id = proto_source.SourceRepository.SourceIdentity.SRCID_UNSPECIFIED with pytest.raises(source_manager.SourceManagerError) as source_error: tree = manager.get_source_tree(source_id) assert not tree assert str(source_error.value) == \ "No Source tree defined for: SRCID_UNSPECIFIED"
def __init__(self, job_control: proto_control.JobControl, benchmark_name: str) -> None: """ Initializes the benchmark class Args: job_control: The protobuf object containing the parameters and locations of benchmark artifacts benchmark_name: The name of the benchmark to execute """ super(Benchmark, self).__init__(job_control, benchmark_name) self._benchmark_dir = None self._envoy_binary_path = job_control.environment.variables[ 'ENVOY_PATH'] self._envoy_builder = None self._nighthawk_builder = None self._source_manager = source_manager.SourceManager(job_control)
def test_get_envoy_hashes_for_benchmark_minimal(mock_run_command): """Verify that we can determine the current and previous image tags from a minimal job control object. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) image_config = _generate_default_benchmark_images(job_control) image_config.envoy_image = "envoyproxy/envoy-dev:latest" mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) hashes = manager.get_envoy_hashes_for_benchmark() assert hashes == {'mocked_hash_the_sequel', 'latest'}
def test_find_all_images_from_specified_tags_using_source( mock_determine_envoy_hashes_from_source): """Verify that return no hashes and if we have to build Envoy""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) # Add an envoy image for us to use as a datum job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0" expected_tags = ['v1.15.2', 'v1.16.0'] mock_determine_envoy_hashes_from_source.return_value = expected_tags manager = source_manager.SourceManager(job_control) tags = manager.find_all_images_from_specified_tags() assert tags == expected_tags
def test_get_source_tree(): """Verify that we can return a source otree object. If no sources are specified, we use the known default location from which to get the source code. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) manager = source_manager.SourceManager(job_control) for source_id in [ proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY, proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK ]: tree = manager.get_source_tree(source_id) assert tree.get_identity() == source_id
def test_get_build_options(): """Verify that we can retrieve specified build options""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_envoy_source(job_control) envoy_source = job_control.source[0] expected_options = ["-c opt", "--jobs 4"] for option in expected_options: envoy_source.bazel_options.add(parameter=option) manager = source_manager.SourceManager(job_control) bazel_options = manager.get_build_options( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) assert bazel_options assert all( [option.parameter in expected_options for option in bazel_options])
def __init__(self, control: proto_control.JobControl) -> None: """Initialize the benchmark object. Perform class member initialization and instantiate the underlying object actually performing the test. Args: control: The Job Control object dictating the parameters governing the benchmark Returns: None """ self._control = control self._source_manager = source_manager.SourceManager(self._control) self._test = [] self._setup_test()
def test_get_build_options_failure(): """Verify that we raise an exception if no options are present in a source repository. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_envoy_source(job_control) manager = source_manager.SourceManager(job_control) bazel_options = None with pytest.raises(source_manager.SourceManagerError) as source_error: bazel_options = manager.get_build_options( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) assert not bazel_options assert str(source_error.value) == \ "No Bazel Options are defined in source: SRCID_ENVOY"
def _prepare_nighthawk(self) -> None: """Prepare the nighthawk source for the benchmark. Checks out the nighthawk source if necessary, builds the client and server binaries """ manager = source_manager.SourceManager(self._control) # This builder needs to be a self object so that the temporary cache # directory is not prematurely cleaned up self._nighthawk_builder = nighthawk_builder.NightHawkBuilder(manager) # FIXME: We build this for each envoy image that we test. self._nighthawk_builder.build_nighthawk_benchmarks() nighthawk_source = manager.get_source_tree( proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK) self._benchmark_dir = nighthawk_source.get_source_directory() log.debug(f"NightHawk benchmark dir {self._benchmark_dir}")
def test_find_all_images_from_specified_sources(mock_copy_source_directory, mock_run_command): """Verify that we can deterimine the previous commit hash from a source tree. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Setup mocks mock_copy_source_directory.return_value = True mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) hashes = manager.find_all_images_from_specified_sources() expected_hashes = { 'expected_previous_commit_hash', 'expected_baseline_hash' } assert hashes == expected_hashes
def test_determine_envoy_hashes_from_source(mock_copy_source_directory, mock_run_command): """ Verify that we can determine Envoy hashes from a source repository """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Setup mocks mock_copy_source_directory.return_value = True mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) hashes = manager.determine_envoy_hashes_from_source() expected_hashes = {'mocked_hash_the_sequel', 'mocked_hash'} assert hashes == expected_hashes
def test_find_all_images_from_specified_tags(): """Verify that we can parse an image tag and deterimine the previous image tag. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) # Add an envoy image and specify additional versions to test job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0" for index in range(1, 4): job_control.images.additional_envoy_images.append( "envoyproxy/envoy:tag{i}".format(i=index)) manager = source_manager.SourceManager(job_control) tags = manager.find_all_images_from_specified_tags() # We test all extra tags first and the specified image tags is always last expected_tags = {'tag1', 'tag2', 'tag3', 'v1.16.0'} assert tags == expected_tags
def test_have_build_options(): """Verify that we can determine if build options exist""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_envoy_source(job_control) envoy_source = job_control.source[0] expected_options = ["-c opt", "--jobs 4"] for option in expected_options: envoy_source.bazel_options.add(parameter=option) # We construct build options for Envoy. NightHawk sources # will not have any options specified manager = source_manager.SourceManager(job_control) bazel_options = manager.have_build_options( proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK) assert not bazel_options bazel_options = manager.have_build_options( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) assert bazel_options
def test_get_image_hashes_from_disk_source(mock_copy_source_directory, mock_run_command): """Verify that we can determine previous hash for a specified commit.""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Setup mocks mock_copy_source_directory.return_value = True mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) tree = manager.get_source_tree( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) hashes = manager.get_image_hashes_from_disk_source( tree, 'expected_baseline_hash') expected_hashes = { 'expected_previous_commit_hash', 'expected_baseline_hash' } assert hashes == expected_hashes