def test_no_source_to_build_nh2(): """Validate that we fail to build nighthawk without sources. Validate that no sources are defined that enable us to build the missing NightHawk binary image. We expect the validation logic to throw an exception """ # create a valid configuration with a missing both NightHawk container images job_control = proto_control.JobControl( remote=False, scavenging_benchmark=True ) images = _generate_images(job_control) images.nighthawk_binary_image = "" images.nighthawk_benchmark_image = "" # Generate a default Envoy source object. Values aren't really checked at # this stage since we have a missing Envoy image, nighthawk source validation # should fail. _generate_envoy_source(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark shoud not throw an exception with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build unspecified NightHawk image"
def _generate_default_job_control() -> proto_control.JobControl: """Generate a default job control object used in tests.""" job_control = proto_control.JobControl( remote=False, scavenging_benchmark=True ) return job_control
def test_no_source_to_build_envoy(): """Validate that we fail to build images without sources. Validate that no sources are present that enable us to build the missing Envoy image We expect the validation logic to throw an exception """ # create a configuration with a missing Envoy image job_control = proto_control.JobControl( remote=False, scavenging_benchmark=True ) images = _generate_images(job_control) images.envoy_image = "" # Denote that the soure is for nighthawk. Values aren't really checked at # this stage since we have a missing Envoy image and a nighthawk source # validation should fail. envoy_source = _generate_envoy_source(job_control) envoy_source.identity = envoy_source.SRCID_NIGHTHAWK benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark shoud not throw an exception with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build unspecified Envoy image"
def test_get_image_hashes_from_disk_source(mock_run_command): """ Verify that we can determine Envoy hashes from source locations """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) source_repo = manager.get_source_repository( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) envoy_source_tree = source_tree.SourceTree(source_repo) expected_hashes = { 'expected_previous_commit_hash', 'expected_baseline_hash' } origin = envoy_source_tree.get_origin() assert origin previous_hash = manager.get_image_hashes_from_disk_source( envoy_source_tree, source_repo.commit_hash) assert previous_hash == expected_hashes
def _load_json_doc(filename: str) -> proto_control.JobControl: """Load a disk file as JSON. This function reads the specified filename and parses the contents as JSON. Args: filename: The file whose contents are to be read as JSON data Returns: A JobControl object populated with the contents from the specified JSON file """ contents = None log.debug(f"Opening JSON file {filename}") try: with open(filename, 'r') as json_doc: contents = json_format.Parse(json_doc.read(), proto_control.JobControl()) except FileNotFoundError as file_not_found: log.exception(f"Unable to load {filename}: {file_not_found}") except json_format.Error as json_parse_error: log.exception( f"Unable to parse JSON contents {filename}: {json_parse_error}") return contents
def test_get_envoy_hashes_for_benchmark_additional_hashes( mock_copy_source_directory, mock_run_command): """Verify that we can determine the hashes for the baseline and previous Envoy Image. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Add an envoy image and specify additional versions to test job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0" for index in range(1, 4): job_control.images.additional_envoy_images.append( "envoyproxy/envoy:tag{i}".format(i=index)) # Setup mocks mock_copy_source_directory.return_value = True mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) hashes = manager.get_envoy_hashes_for_benchmark() # Since both a source and image was specified, we benchmark # the source at its current and previous commit, as well as # the other specified image tags. expected_hashes = { 'tag1', 'tag2', 'tag3', 'v1.16.0', 'expected_previous_commit_hash', 'expected_baseline_hash' } assert hashes == expected_hashes
def test_get_image_hashes_from_disk_source_fail2( mock_get_previous_commit_hash): """Verify that we raise an exception if we are not able to determine the prior hash to a specified commit.""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Setup mocks mock_get_previous_commit_hash.side_effect = _raise_source_tree_error manager = source_manager.SourceManager(job_control) tree = manager.get_source_tree( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) hashes = {} with pytest.raises(source_manager.SourceManagerError) as source_error: hashes = manager.get_image_hashes_from_disk_source( tree, 'expected_baseline_hash') assert not hashes assert str(source_error.value) == \ "Unable to find a commit hash prior to [expected_baseline_hash]"
def test_determine_envoy_hashes_from_source2(mock_source_tree_pull, mock_run_command): """ Verify that we can determine Envoy hashes from a source repository This test exercises the else case where we use the head hash instead of a specific envoy tag. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Add an envoy image and specify additional versions to test job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0" # Setup mocks mock_source_tree_pull.return_value = True mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) hashes = manager.determine_envoy_hashes_from_source() expected_hashes = {'v1.15.2', 'v1.16.0'} assert hashes == expected_hashes
def _load_yaml_doc(filename: str) -> proto_control.JobControl: """Load a disk file as YAML. This function reads the specified filename and parses the contents as YAML. Args: filename: The file whose contents are to be read as YAML data Returns: A JobControl object populated with the contents from the specified YAML file """ log.debug(f"Opening YAML file {filename}") contents = None try: with open(filename, 'r') as yaml_doc: contents = yaml.load(yaml_doc.read()) contents = json_format.Parse( json.dumps(contents), proto_control.JobControl()) except FileNotFoundError as file_not_found: log.exception(f"Unable to load {filename}: {file_not_found}") except json_format.Error as yaml_parse_error: log.exception( f"Unable to parse YAML contents {filename}: {yaml_parse_error}") return contents
def test_determine_envoy_hashes_from_source_pull_fail( mock_copy_source_directory, mock_source_tree_pull): """ Verify that an exception is raised when we cannot determine Envoy hashes from a job control object """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Setup mocks to simulate a source retrieval failure mock_copy_source_directory.return_value = False mock_source_tree_pull.return_value = False manager = source_manager.SourceManager(job_control) hashes = None with pytest.raises(source_manager.SourceManagerError) as source_error: hashes = manager.determine_envoy_hashes_from_source() assert not hashes assert str(source_error.value) == \ "Unable to obtain the source to determine commit hashes"
def test_execute_dockerized_benchmark_using_images_only( mock_hashes_for_benchmarks, mock_have_build_options, mock_pull_image, mock_execute, mock_run_image, mock_symlink): """Verify that we attempt to pull images if no sources are specified.""" # Build a default job control object with images job_control = proto_control.JobControl(remote=False, dockerized_benchmark=True) generate_test_objects.generate_environment(job_control) generate_test_objects.generate_images(job_control) mock_run_image.return_value = b"benchmark_http_client output...." mock_execute.return_value = None mock_have_build_options.return_value = False mock_hashes_for_benchmarks.return_value = {'tag1', 'tag2'} # Instantiate the BenchmarkRunner so that it prepares the job control # objects for each benchmark benchmark = run_benchmark.BenchmarkRunner(job_control) benchmark.execute() mock_have_build_options.assert_called() mock_pull_image.assert_called() mock_symlink.assert_called() mock_execute.assert_has_calls([mock.call(), mock.call()])
def _create_new_job_control(self, envoy_image) -> proto_control.JobControl: """Duplicate the job control for a specific benchmark run. This method creates a new job control object and sets the commit hash for the envoy revision being tested Args: envoy_image: The envoy image name being tested. This is expected to be in the format "envoyproxy/envoy-dev:tag". Returns: A job control document containing the hash and image name being tested """ image_hash = envoy_image.split(':')[-1] output_dir = os.path.join(self._control.environment.output_dir, image_hash) new_job_control = proto_control.JobControl() new_job_control.CopyFrom(self._control) new_job_control.images.envoy_image = envoy_image new_job_control.environment.output_dir = output_dir self._create_symlink_for_test_artifacts(output_dir, image_hash) return new_job_control
def _generate_default_source_manager(): """Build a default SourceRepository object.""" control = proto_control.JobControl(remote=False, scavenging_benchmark=True) control.source.add( identity=proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK, source_path='/where_nighthawk_code_lives', ) return source_manager.SourceManager(control)
def _generate_default_source_manager(): """Build a default SourceRepository object.""" control = proto_control.JobControl(remote=False, scavenging_benchmark=True) control.source.add( identity=proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY, source_path='/some_random_envoy_directory', commit_hash='v1.16.0') return source_manager.SourceManager(control)
def test_binary_benchmark_setup(mock_have_build_options, mock_symlink): """Verify that the unique methods to the binary benchmark workflow are in order""" job_control = proto_control.JobControl(remote=False, binary_benchmark=True) generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_nighthawk_source(job_control) benchmark = run_benchmark.BenchmarkRunner(job_control) mock_symlink.assert_called_with( 'source_url__hash_doesnt_really_matter_here__master', 'source_url__hash_doesnt_really_matter_here__master')
def test_find_all_images_from_specified_tags_build_envoy(): """Verify that return no hashes and if we have to build Envoy""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) manager = source_manager.SourceManager(job_control) tags = manager.find_all_images_from_specified_tags() # Since the envoy image is not specified, we have not tags for a datum expected_tags = set() assert tags == expected_tags
def generate_image_manager_with_source_url(): """Generate a source manager with a job control specifying remote repos for images. """ job_control = proto_control.JobControl() job_control.source.add( identity=proto_source.SourceRepository.SRCID_ENVOY, source_url='https://www.github.com/_some_random_repo_') job_control.source.add( identity=proto_source.SourceRepository.SRCID_NIGHTHAWK, source_url='https://www.github.com/_nighthawk_repo_') return source_manager.SourceManager(job_control)
def test_benchmark_failure_if_no_benchmark_selected(): """Verify that we raise an exception if no benchmark is configured to run. """ # Build a default job control object no benchmark selected job_control = proto_control.JobControl(remote=False) # Instantiate the BenchmarkRunner so that it prepares the job control # objects for each benchmark with pytest.raises(NotImplementedError) as not_implemented: _ = run_benchmark.BenchmarkRunner(job_control) assert str(not_implemented.value) == \ "No [Unspecified Benchmark] defined"
def test_find_all_images_from_specified_tags_fail(): """Verify that we raise an exception if no images are defined for any benchmarks""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) manager = source_manager.SourceManager(job_control) hashes = [] with pytest.raises(source_manager.SourceManagerError) as source_error: hashes = manager.find_all_images_from_specified_tags() assert not hashes assert str(source_error.value) == \ "No images are specified in the control document"
def test_validate_must_be_overidden(): """Verify that the base _validate method must be overridden and raises an exception otherwise. """ control = proto_control.JobControl(remote=False, scavenging_benchmark=True) control.source.add( identity=proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY, source_path='/some_random_envoy_directory', ) manager = source_manager.SourceManager(control) builder = DerivedBuilder(manager) with pytest.raises(NotImplementedError) as not_implemented: builder.do_something() assert str(not_implemented.value) == "Method should be overridden"
def test_find_all_images_from_specified_tags_using_source( mock_determine_envoy_hashes_from_source): """Verify that return no hashes and if we have to build Envoy""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) # Add an envoy image for us to use as a datum job_control.images.envoy_image = "envoyproxy/envoy:v1.16.0" expected_tags = ['v1.15.2', 'v1.16.0'] mock_determine_envoy_hashes_from_source.return_value = expected_tags manager = source_manager.SourceManager(job_control) tags = manager.find_all_images_from_specified_tags() assert tags == expected_tags
def test_get_envoy_hashes_for_benchmark_minimal(mock_run_command): """Verify that we can determine the current and previous image tags from a minimal job control object. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) image_config = _generate_default_benchmark_images(job_control) image_config.envoy_image = "envoyproxy/envoy-dev:latest" mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) hashes = manager.get_envoy_hashes_for_benchmark() assert hashes == {'mocked_hash_the_sequel', 'latest'}
def test_get_source_tree(): """Verify that we can return a source otree object. If no sources are specified, we use the known default location from which to get the source code. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) manager = source_manager.SourceManager(job_control) for source_id in [ proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY, proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK ]: tree = manager.get_source_tree(source_id) assert tree.get_identity() == source_id
def test_get_source_tree_fail(): """Verify that we raise an assertion if we are not able to find a source repository. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) manager = source_manager.SourceManager(job_control) tree = None source_id = proto_source.SourceRepository.SourceIdentity.SRCID_UNSPECIFIED with pytest.raises(source_manager.SourceManagerError) as source_error: tree = manager.get_source_tree(source_id) assert not tree assert str(source_error.value) == \ "No Source tree defined for: SRCID_UNSPECIFIED"
def _create_new_source_job_control(self, nh_source, envoy_source, envoy_hash) \ -> proto_control.JobControl: """Duplicate the job control for a specific benchmark run. This method creates a new job control object for a single binary benchmark Args: nh_source: the nighthawk source to run envoy_source: the envoy source to test Returns: A job control document containing the Nighthawk and Envoy source specified """ # Generate a unique identifier for the individual run # 1. Specifies whether source is provided via source_path or source_url # 2. Specifies the primary commit hash or tag, if provided # 3. Specifies the branch, if provided # Ex: source_url__v1.16.0__branch_name identifier = str(envoy_source.WhichOneof('source_location')) if envoy_hash: identifier += '__' identifier += envoy_hash if envoy_source.branch: identifier += '__' identifier += envoy_source.branch # Create a new SourceRepository representing just one Envoy build target new_envoy_source = proto_source.SourceRepository() new_envoy_source.CopyFrom(envoy_source) new_envoy_source.commit_hash = envoy_hash del new_envoy_source.additional_hashes[:] output_dir = os.path.join(self._control.environment.output_dir, identifier) new_job_control = proto_control.JobControl() new_job_control.CopyFrom(self._control) del new_job_control.source[:] new_job_control.source.extend([nh_source, new_envoy_source]) new_job_control.environment.output_dir = output_dir log.debug(f"Creating new Binary job for {new_envoy_source}") log.debug(f"Job:\n{new_job_control}") self._create_symlink_for_test_artifacts(output_dir, identifier) return new_job_control
def test_get_build_options(): """Verify that we can retrieve specified build options""" job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_envoy_source(job_control) envoy_source = job_control.source[0] expected_options = ["-c opt", "--jobs 4"] for option in expected_options: envoy_source.bazel_options.add(parameter=option) manager = source_manager.SourceManager(job_control) bazel_options = manager.get_build_options( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) assert bazel_options assert all( [option.parameter in expected_options for option in bazel_options])
def test_no_valid_envoy_binary(mock_nh_bin_build, mock_nh_bench_build): """Validate that we fail when Envoy sources are not present, and no binary is specified. We expect an unhandled exception to surface from _prepare_envoy() """ # create a valid configuration with a missing both NightHawk container images job_control = proto_control.JobControl(remote=False, binary_benchmark=True) job_control.environment.variables['ENVOY_PATH'] = '/dev/null/foo' generate_test_objects.generate_nighthawk_source(job_control) generate_test_objects.generate_environment(job_control) benchmark = binary_benchmark.Benchmark(job_control, "test_benchmark") with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "ENVOY_PATH environment variable specified, but invalid"
def test_get_build_options_failure(): """Verify that we raise an exception if no options are present in a source repository. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_envoy_source(job_control) manager = source_manager.SourceManager(job_control) bazel_options = None with pytest.raises(source_manager.SourceManagerError) as source_error: bazel_options = manager.get_build_options( proto_source.SourceRepository.SourceIdentity.SRCID_ENVOY) assert not bazel_options assert str(source_error.value) == \ "No Bazel Options are defined in source: SRCID_ENVOY"
def test_execute_benchmark_missing_nighthawk_binary_image(): """Validate that no sources are defined that enable us to build the missing NightHawk benchmark image resulting in a raised exception. """ # create a valid configuration with a missing NightHawk container image job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) images = generate_test_objects.generate_images(job_control) images.nighthawk_binary_image = "" # Generate a default Envoy source object. generate_test_objects.generate_envoy_source(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark raise an exception from validate() with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build NightHawk image"
def test_find_all_images_from_specified_sources(mock_copy_source_directory, mock_run_command): """Verify that we can deterimine the previous commit hash from a source tree. """ job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) _generate_default_benchmark_images(job_control) _generate_default_envoy_source(job_control) # Setup mocks mock_copy_source_directory.return_value = True mock_run_command.side_effect = _run_command_side_effect manager = source_manager.SourceManager(job_control) hashes = manager.find_all_images_from_specified_sources() expected_hashes = { 'expected_previous_commit_hash', 'expected_baseline_hash' } assert hashes == expected_hashes