def execute_benchmark(self) -> None: """Execute the binary benchmark Uses either the Envoy specified in ENVOY_PATH, or one built from a specified source. """ self._validate() self._prepare_nighthawk() self._prepare_envoy() #todo: refactor args, have frontend specify them via protobuf cmd = ("bazel test " "--test_summary=detailed " "--test_output=all " "--test_arg=--log-cli-level=info " "--test_env=ENVOY_IP_TEST_VERSIONS=v4only " "--test_env=HEAPPROFILE= " "--test_env=HEAPCHECK= " "--cache_test_results=no " "--compilation_mode=opt " "--cxxopt=-g " "--cxxopt=-ggdb3 " "--define tcmalloc=gperftools " "//benchmarks:* ") cmd_params = cmd_exec.CommandParameters(cwd=self._benchmark_dir) # pull in environment and set values env = self._control.environment # 'TMPDIR' is required for successful operation. This is the output # directory for all produced NightHawk artifacts binary_benchmark_vars = {'TMPDIR': env.output_dir} if self._envoy_binary_path: binary_benchmark_vars['ENVOY_PATH'] = self._envoy_binary_path log.debug(f"Using environment: {binary_benchmark_vars}") for (key, value) in binary_benchmark_vars.items(): if key not in env.variables: log.debug( f"Building control environment variables: {key}={value}") env.variables[key] = value environment_controller = base_benchmark.BenchmarkEnvController(env) with environment_controller: try: cmd_exec.run_command(cmd, cmd_params) except subprocess.CalledProcessError as cpe: log.error(f"Unable to execute the benchmark: {cpe}")
def test_run_command(mock_check_call): """Verify that we can return the output from a check_call call.""" mock_check_call.side_effect = check_call_side_effect cmd_parameters = cmd_exec.CommandParameters(cwd='/tmp') cmd = 'spanish_output_stdout' output = cmd_exec.run_command(cmd, cmd_parameters) assert output == 'No te hablas una palabra del espanol' cmd = 'spanish_output_stderr' output = cmd_exec.run_command(cmd, cmd_parameters) assert output == 'No te hablas una palabra del espanol en stderr'
def create_docker_image(self) -> None: """Build a docker image with the newly compiled Envoy binary.""" self._generate_docker_ignore() commit_hash = self._source_repo.commit_hash cmd = "docker build " cmd += "-f ci/Dockerfile-envoy " cmd += "-t envoyproxy/envoy-dev:{hash} ".format(hash=commit_hash) cmd += "--build-arg TARGETPLATFORM=\'.\' ." cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir) cmd_exec.run_command(cmd, cmd_params)
def checkout_commit_hash(self) -> bool: """Checks out the specified commit hash in the source tree Returns: a boolean indicating whether the operation was successful """ self._validate() # Assume the checkout is successful. It is possible that it is a no-op # if no commit hashes are specified checkout_success = True # Clone the repo if it doesn't exist on disk if not self._source_repo.source_path and self._source_repo.source_url: log.debug( "No local source path exists. Cloning repo for hash discovery" ) self.pull() if self._source_repo.commit_hash: cmd = "git checkout {hash}".format( hash=self._source_repo.commit_hash) cmd_params = cmd_exec.CommandParameters( cwd=self.get_source_directory()) output = cmd_exec.run_command(cmd, cmd_params) # HEAD is now at <8 chars of hash> expected = "HEAD is now at {commit_hash}".format( commit_hash=self._source_repo.commit_hash[:8]) checkout_success = expected in output return checkout_success
def _run_bazel_clean(self) -> None: """Run bazel clean in the source tree directory.""" assert self._build_dir cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir) cmd = "bazel clean" output = cmd_exec.run_command(cmd, cmd_params) log.debug(f"Clean output: {output}")
def execute_benchmark(self) -> None: """Execute the scavenging benchmark. Raises: BenchmarkError: if the benchmark fails to execute successfully """ self._validate() self._prepare_nighthawk() # pull in environment and set values env = self._control.environment output_dir = env.output_dir images = self.get_images() log.debug(f"Images: {images.nighthawk_benchmark_image}") # 'TMPDIR' is required for successful operation. This is the output # directory for all produced NightHawk artifacts image_vars = { 'NH_DOCKER_IMAGE': images.nighthawk_binary_image, 'ENVOY_DOCKER_IMAGE_TO_TEST': images.envoy_image, 'TMPDIR': output_dir } log.debug(f"Using environment: {image_vars}") for (key, value) in image_vars.items(): if key not in env.variables: log.debug( f"Building control environment variables: {key}={value}") env.variables[key] = value environment_controller = base_benchmark.BenchmarkEnvController(env) cmd = ("bazel-bin/benchmarks/benchmarks " "--log-cli-level=info -vvvv -k test_http_h1_small " "benchmarks/") cmd_params = cmd_exec.CommandParameters(cwd=self._benchmark_dir) with environment_controller: try: cmd_exec.run_command(cmd, cmd_params) except subprocess.CalledProcessError as cpe: raise base_benchmark.BenchmarkError( f"Unable to execute the benchmark: {cpe}")
def get_previous_commit_hash(self, current_commit: str, revisions: int = 2) -> str: """Return the specified number of commits behind the current commit hash. Args: current_commit: The current hash from which we are starting the search revisions: The number of commits in the tree that we skip over, starting from the current_commit Returns: a string with the discovered commit hash Raises: SourceTreeError if we are not able to deduce the previous commit """ assert current_commit if not self.pull(): log.debug("Source pull failed. Copying source directory") self.copy_source_directory() log.debug( f"Finding previous commit to current commit: [{current_commit}]") if is_tag(current_commit): log.info(f"Current commit \"{current_commit}\" is a tag.") return self.get_previous_tag(current_commit) if current_commit == 'latest': current_commit = self.get_head_hash() cmd = "git rev-list --no-merges --committer='GitHub <*****@*****.**>' " cmd += "--max-count={revisions} {commit}".format(revisions=revisions, commit=current_commit) cmd_params = cmd_exec.CommandParameters( cwd=self.get_source_directory()) hash_list = cmd_exec.run_command(cmd, cmd_params) # Check whether we got an error from git if 'unknown revision or path not in the working tree.' in hash_list: raise SourceTreeError(hash_list) # Reverse iterate throught the list of hashes, skipping any blank # lines that may have trailed the original git output for commit_hash in hash_list.split('\n')[::-1]: if commit_hash: log.debug(f"Returning {commit_hash} as the previous commit to " f"{current_commit}") return commit_hash raise SourceTreeError(f"No commit found prior to {current_commit}")
def stage_envoy(self, strip_binary: bool) -> None: """Copy and optionally strip the Envoy binary. After we compile Envoy, copy the binary into a platform directory for inclusion in the docker image. It is unclear the intent of the 'targetplatform' parameter in the Dockerfile, so we use a static string. Ultimately the compiled binary is staged for packaging in the resulting image. Args: strip_binary: determines whether we use objcopy to strip debug symbols from the envoy binary. If strip_binary is False, we simply copy the binary to its destination Callers use False for now, until we expose a control for manipulating this parameter. Returns: None """ # Stage the envoy binary for the docker image dir_mode = 0o755 pwd = os.getcwd() os.chdir(self._build_dir) if not os.path.exists('build_release_stripped'): os.mkdir('build_release_stripped', dir_mode) os.chdir(pwd) cmd = "objcopy --strip-debug " if strip_binary else "cp -fv " cmd += constants.ENVOY_BINARY_TARGET_OUTPUT_PATH cmd += " build_release_stripped/envoy" cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir) cmd_exec.run_command(cmd, cmd_params)
def build_nighthawk_binaries(self) -> None: """Build the NightHawk client and server binaries. This is a pre-requisite to building the nighthawk binary docker image """ self.prepare_nighthawk_source() cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir) bazel_options = self._generate_bazel_options( proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK) cmd = "bazel build {bazel_options} //:nighthawk".format( bazel_options=bazel_options) output = cmd_exec.run_command(cmd, cmd_params) log.debug(f"Nighthawk build output: {output}")
def test_run_command_fail(mock_check_call): """Verify that a CalledProcessError is bubbled to the caller if the command fails. """ mock_check_call.side_effect = check_call_side_effect cmd_parameters = cmd_exec.CommandParameters(cwd='/tmp') cmd = 'command_error' output = '' with pytest.raises(subprocess.CalledProcessError) as process_error: output = cmd_exec.run_command(cmd, cmd_parameters) assert not output assert f"Command \'{cmd}\' returned non-zero exit status" in \ str(process_error.value)
def get_head_hash(self) -> str: """Retrieve the hash for the HEAD commit. Returns: a string containing the hash corresponding to commit at the HEAD of the tree. """ self._validate() cmd = ( "git rev-list --no-merges --committer='GitHub <*****@*****.**>' " "--max-count=1 HEAD") cmd_params = cmd_exec.CommandParameters( cwd=self.get_source_directory()) return cmd_exec.run_command(cmd, cmd_params)
def list_tags(self) -> List[str]: """Enumerate the repository tags and return them in a list. Returns: a list of tags from the commits """ self._validate() cmd = "git tag --list --sort v:refname" cmd_params = cmd_exec.CommandParameters( cwd=self.get_source_directory()) tag_output = cmd_exec.run_command(cmd, cmd_params) tag_list = [tag.strip() for tag in tag_output.split('\n') if tag] log.debug(f"Repository tags {tag_list}") return tag_list
def build_nighthawk_benchmarks(self) -> None: """Build the NightHawk benchmarks target. This target is required for the scavenging benchmark. It is also a pre- requisite to building the benchmark container image """ self.prepare_nighthawk_source() cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir) bazel_options = self._generate_bazel_options( proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK) cmd = "bazel build {bazel_options} //benchmarks:benchmarks".format( bazel_options=bazel_options) output = cmd_exec.run_command(cmd, cmd_params) log.debug(f"Nighthawk build output: {output}")
def get_revs_behind_parent_branch(self) -> int: """Get the number of commits behind the parent branch. Determine how many commits the current branch on disk is behind the parent branch. If we are up to date, return zero Returns: an integer with the number of commits the local source lags behind the parent branch """ self._validate() cmd = "git status" output_directory = self.get_source_directory() cmd_params = cmd_exec.CommandParameters(cwd=output_directory) status_output = cmd_exec.run_command(cmd, cmd_params) commit_count = 0 # Extract the commit count from lines such as: # # Your branch is ahead of 'origin/master' by 99 commits. # # or determine whether git believes we are up to date: # # Your branch is up to date with 'origin/master'. ahead = re.compile(_REPO_STATUS_REGEX) up_to_date = re.compile(r'Your branch is up to date with \'(.*)\'') for line in status_output.split('\n'): match = ahead.match(line) if match: commit_count = int(match.group(2)) log.debug( f"Branch is {commit_count} ahead of branch {match.group(1)}" ) break match = up_to_date.match(line) if match: log.debug(f"Branch {match.group(1)} is up to date") break return commit_count
def _execute_docker_image_script(script: str, build_dir: str) -> None: """Run the specified script to build a docker image. The docker image tags are "fixed" at "latest" for the binary container. The benchmark image's tag can be adjusted using DOCKER_IMAGE_TAG however this value defaults to "latest" as well. We are not currently exposing a method to set this environment variable. When buliding the nighthhawk components we use the most recent source by default. Args: script: The shell script in the nighthawk repository that builds the benchmark and binary docker images. build_dir: The nighthawk source location """ cmd_params = cmd_exec.CommandParameters(cwd=build_dir) output = cmd_exec.run_command(script, cmd_params) log.debug(f"NightHawk Docker image output for {script}: {output}")
def pull(self) -> bool: """Retrieve the code from the repository. Uses git to clone the source into a working directory that has read/write permissions by salvo Returns: a boolean indicating whether the operation was successful """ self._validate() source_name = proto_source.SourceRepository.SourceIdentity.Name( self._source_repo.identity) log.debug(f"Pulling [{source_name}] from origin: " f"[{self._source_repo.source_url}]") if not self._source_repo.source_url: log.debug("No url specified for source. Cannot pull.") return False try: if self.is_up_to_date(): return True except subprocess.CalledProcessError: log.info("Source likely does not exist on disk") if not self._source_repo.source_url: self._source_repo.source_url = self.get_origin() # Clone into the working directory cmd = "git clone {origin} .".format( origin=self._source_repo.source_url) cmd_params = cmd_exec.CommandParameters( cwd=self.get_source_directory()) output = cmd_exec.run_command(cmd, cmd_params) expected = 'Cloning into \'.\'' return expected in output
def get_origin(self) -> str: """Detect the origin url from where the code is fetched. Returns: A string showing the origin url for the source tree. This needed most for remote execution where we do not ship the entire source tree remotely. We will attempt to generate a patch between the origin HEAD and the local HEAD. This diff is applied to the source in the remote context. Raises: SourceTreeError: if we are not able to determine the origin url for a managed source tree """ valid = self._validate() log.debug(f"Valid: {valid} for object: {self}") origin_url = self._source_repo.source_url output_directory = self.get_source_directory() cmd = "git remote -v" if not origin_url: cmd_params = cmd_exec.CommandParameters(cwd=output_directory) output = cmd_exec.run_command(cmd, cmd_params) for line in output.split('\n'): match = re.match(_GIT_ORIGIN_REGEX, line) if match: origin_url = match.group(1) self._source_repo.source_url = origin_url break if not origin_url: raise SourceTreeError( f"Unable to determine the origin url from {output_directory}") return origin_url