def test_images_only_config(): """Test benchmark validation logic.""" # create a valid configuration defining images only for benchmark job_control = _generate_default_job_control() _generate_images(job_control) _generate_environment(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") run_parameters = docker_image.DockerRunParameters( command=['./benchmarks', '--log-cli-level=info', '-vvvv'], environment=mock.ANY, volumes=mock.ANY, network_mode='host', tty=True ) # Calling execute_benchmark shoud not throw an exception # Mock the container invocation so that we don't try to pull an image with mock.patch('src.lib.benchmark.base_benchmark.BaseBenchmark.run_image') \ as docker_mock: benchmark.execute_benchmark() docker_mock.assert_called_once_with( 'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag', run_parameters)
def test_execute_benchmark_with_envoy_source(mock_run_image): """Validate that if sources are defined that enable us to build the Envoy image we do not throw an exception. """ mock_run_image.return_value = b'benchmark_http_client output' # create a valid configuration with a missing Envoy image job_control = generate_test_objects.generate_default_job_control() images = generate_test_objects.generate_images(job_control) images.envoy_image = "" generate_test_objects.generate_envoy_source(job_control) generate_test_objects.generate_environment(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") run_parameters = docker_image.DockerRunParameters( command=['./benchmarks', '--log-cli-level=info', '-vvvv'], environment=mock.ANY, volumes=mock.ANY, network_mode='host', tty=True) benchmark.execute_benchmark() mock_run_image.assert_called_once_with( 'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag', run_parameters)
def test_execute_benchmark_using_images_only(mock_run_image): """Validate we execute the benchmark with only images specified.""" mock_run_image.return_value = b'benchmark_http_client output' # create a valid configuration defining images only for benchmark job_control = generate_test_objects.generate_default_job_control() generate_test_objects.generate_images(job_control) generate_test_objects.generate_environment(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") run_parameters = docker_image.DockerRunParameters( command=['./benchmarks', '--log-cli-level=info', '-vvvv'], environment=mock.ANY, volumes=mock.ANY, network_mode='host', tty=True) # Calling execute_benchmark shoud not throw an exception benchmark.execute_benchmark() mock_run_image.assert_called_once_with( 'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag', run_parameters)
def test_no_source_to_build_nh2(): """Validate that we fail to build nighthawk without sources. Validate that no sources are defined that enable us to build the missing NightHawk binary image. We expect the validation logic to throw an exception """ # create a valid configuration with a missing both NightHawk container images job_control = proto_control.JobControl( remote=False, scavenging_benchmark=True ) images = _generate_images(job_control) images.nighthawk_binary_image = "" images.nighthawk_benchmark_image = "" # Generate a default Envoy source object. Values aren't really checked at # this stage since we have a missing Envoy image, nighthawk source validation # should fail. _generate_envoy_source(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark shoud not throw an exception with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build unspecified NightHawk image"
def test_no_source_to_build_envoy(): """Validate that we fail to build images without sources. Validate that no sources are present that enable us to build the missing Envoy image We expect the validation logic to throw an exception """ # create a configuration with a missing Envoy image job_control = proto_control.JobControl( remote=False, scavenging_benchmark=True ) images = _generate_images(job_control) images.envoy_image = "" # Denote that the soure is for nighthawk. Values aren't really checked at # this stage since we have a missing Envoy image and a nighthawk source # validation should fail. envoy_source = _generate_envoy_source(job_control) envoy_source.identity = envoy_source.SRCID_NIGHTHAWK benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark shoud not throw an exception with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build unspecified Envoy image"
def _setup_test(self) -> None: """Create the object performing the desired benchmark. Instantiate the object performing the actual test and setup the control objects for each test invocation. Raises: NotImplementedError: for tests and/or modes that are not yet implemented. """ current_benchmark_name = "Unspecified Benchmark" if self._control.scavenging_benchmark: current_benchmark_name = "Scavenging Benchmark" job_control_list = self._generate_job_control_for_envoy_images() for job_control in job_control_list: benchmark = scavenging.Benchmark(job_control, current_benchmark_name) self._test.append(benchmark) elif self._control.dockerized_benchmark: current_benchmark_name = "Fully Dockerized Benchmark" job_control_list = self._generate_job_control_for_envoy_images() for job_control in job_control_list: benchmark = fulldocker.Benchmark(job_control, current_benchmark_name) self._test.append(benchmark) elif self._control.binary_benchmark: current_benchmark_name = "Binary Benchmark" # Not working with docker images here, so use custom binary-oriented job control generation job_control_list = self._generate_job_control_for_binaries() for job_control in job_control_list: benchmark = binary_benchmark.Benchmark(job_control, current_benchmark_name) self._test.append(benchmark) if not self._test: raise NotImplementedError(f"No [{current_benchmark_name}] defined")
def test_execute_benchmark_no_image_or_sources(): """Verify that the validation logic raises an exception since we are unable to build a required Envoy image. """ # create a valid configuration with a missing Envoy image job_control = generate_test_objects.generate_default_job_control() images = generate_test_objects.generate_images(job_control) images.envoy_image = "" generate_test_objects.generate_environment(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark shoud not throw an exception with pytest.raises(base_benchmark.BenchmarkError) \ as validation_error: benchmark.execute_benchmark() assert str(validation_error.value) == "No source configuration specified"
def test_execute_benchmark_missing_nighthawk_binary_image(): """Validate that no sources are defined that enable us to build the missing NightHawk benchmark image resulting in a raised exception. """ # create a valid configuration with a missing NightHawk container image job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) images = generate_test_objects.generate_images(job_control) images.nighthawk_binary_image = "" # Generate a default Envoy source object. generate_test_objects.generate_envoy_source(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark raise an exception from validate() with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build NightHawk image"
def test_execute_benchmark_missing_nighthawk_benchmark_image(): """Validate an exception is raised if we cannot build the unspecified NightHawk benchmark image. """ # create a valid configuration with a missing both NightHawk container images job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) images = generate_test_objects.generate_images(job_control) images.nighthawk_benchmark_image = "" # Generate a default Envoy source object generate_test_objects.generate_envoy_source(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark shoud not throw an exception with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build NightHawk image"
def test_no_envoy_image_no_sources(): """Test benchmark validation logic. No Envoy image is specified, we expect the validation logic to throw an exception since no sources are present """ # create a valid configuration with a missing Envoy image job_control = _generate_default_job_control() images = _generate_images(job_control) images.envoy_image = "" _generate_environment(job_control) benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark shoud not throw an exception with pytest.raises(full_docker.FullyDockerizedBenchmarkError) \ as validation_error: benchmark.execute_benchmark() assert str(validation_error.value) == "No source configuration specified"
def test_execute_benchmark_missing_envoy_source(): """Validate that although sources are defined for NightHawk we raise an exception due to the inability to build Envoy. """ # create a configuration with a missing Envoy image job_control = proto_control.JobControl(remote=False, scavenging_benchmark=True) images = generate_test_objects.generate_images(job_control) images.envoy_image = "" # Change the source identity to NightHawk envoy_source = generate_test_objects.generate_envoy_source(job_control) envoy_source.identity = envoy_source.SRCID_NIGHTHAWK benchmark = full_docker.Benchmark(job_control, "test_benchmark") # Calling execute_benchmark should raise an exception from validate() with pytest.raises(Exception) as validation_exception: benchmark.execute_benchmark() assert str(validation_exception.value) == \ "No source specified to build Envoy image"
def test_source_to_build_envoy(): """Validate we can build images from source. Validate that sources are defined that enable us to build the Envoy image We do not expect the validation logic to throw an exception """ # create a valid configuration with a missing Envoy image job_control = _generate_default_job_control() images = _generate_images(job_control) images.envoy_image = "" _generate_envoy_source(job_control) env = job_control.environment env.test_version = env.IPV_V4ONLY env.envoy_path = 'envoy' env.output_dir = 'some_output_dir' env.test_dir = 'some_test_dir' benchmark = full_docker.Benchmark(job_control, "test_benchmark") run_parameters = docker_image.DockerRunParameters( command=['./benchmarks', '--log-cli-level=info', '-vvvv'], environment=mock.ANY, volumes=mock.ANY, network_mode='host', tty=True ) # Mock the container invocation so that we don't try to pull an image with mock.patch('src.lib.benchmark.base_benchmark.BaseBenchmark.run_image') \ as docker_mock: benchmark.execute_benchmark() docker_mock.assert_called_once_with( 'envoyproxy/nighthawk-benchmark-dev:random_benchmark_image_tag', run_parameters)