def execute_benchmark(self) -> None:
"""Execute the binary benchmark
Uses either the Envoy specified in ENVOY_PATH, or one built from a
specified source.
"""
self._validate()
self._prepare_nighthawk()
self._prepare_envoy()
#todo: refactor args, have frontend specify them via protobuf
cmd = ("bazel test "
"--test_summary=detailed "
"--test_output=all "
"--test_arg=--log-cli-level=info "
"--test_env=ENVOY_IP_TEST_VERSIONS=v4only "
"--test_env=HEAPPROFILE= "
"--test_env=HEAPCHECK= "
"--cache_test_results=no "
"--compilation_mode=opt "
"--cxxopt=-g "
"--cxxopt=-ggdb3 "
"--define tcmalloc=gperftools "
"//benchmarks:* ")
cmd_params = cmd_exec.CommandParameters(cwd=self._benchmark_dir)
# pull in environment and set values
env = self._control.environment
# 'TMPDIR' is required for successful operation. This is the output
# directory for all produced NightHawk artifacts
binary_benchmark_vars = {'TMPDIR': env.output_dir}
if self._envoy_binary_path:
binary_benchmark_vars['ENVOY_PATH'] = self._envoy_binary_path
log.debug(f"Using environment: {binary_benchmark_vars}")
for (key, value) in binary_benchmark_vars.items():
if key not in env.variables:
log.debug(
f"Building control environment variables: {key}={value}")
env.variables[key] = value
environment_controller = base_benchmark.BenchmarkEnvController(env)
with environment_controller:
try:
cmd_exec.run_command(cmd, cmd_params)
except subprocess.CalledProcessError as cpe:
log.error(f"Unable to execute the benchmark: {cpe}")
def test_run_command(mock_check_call): """Verify that we can return the output from a check_call call.""" mock_check_call.side_effect = check_call_side_effect cmd_parameters = cmd_exec.CommandParameters(cwd='/tmp') cmd = 'spanish_output_stdout' output = cmd_exec.run_command(cmd, cmd_parameters) assert output == 'No te hablas una palabra del espanol' cmd = 'spanish_output_stderr' output = cmd_exec.run_command(cmd, cmd_parameters) assert output == 'No te hablas una palabra del espanol en stderr'
def create_docker_image(self) -> None:
"""Build a docker image with the newly compiled Envoy binary."""
self._generate_docker_ignore()
commit_hash = self._source_repo.commit_hash
cmd = "docker build "
cmd += "-f ci/Dockerfile-envoy "
cmd += "-t envoyproxy/envoy-dev:{hash} ".format(hash=commit_hash)
cmd += "--build-arg TARGETPLATFORM=\'.\' ."
cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir)
cmd_exec.run_command(cmd, cmd_params)
def checkout_commit_hash(self) -> bool:
"""Checks out the specified commit hash in the source tree
Returns:
a boolean indicating whether the operation was successful
"""
self._validate()
# Assume the checkout is successful. It is possible that it is a no-op
# if no commit hashes are specified
checkout_success = True
# Clone the repo if it doesn't exist on disk
if not self._source_repo.source_path and self._source_repo.source_url:
log.debug(
"No local source path exists. Cloning repo for hash discovery"
)
self.pull()
if self._source_repo.commit_hash:
cmd = "git checkout {hash}".format(
hash=self._source_repo.commit_hash)
cmd_params = cmd_exec.CommandParameters(
cwd=self.get_source_directory())
output = cmd_exec.run_command(cmd, cmd_params)
# HEAD is now at <8 chars of hash>
expected = "HEAD is now at {commit_hash}".format(
commit_hash=self._source_repo.commit_hash[:8])
checkout_success = expected in output
return checkout_success
def _run_bazel_clean(self) -> None:
"""Run bazel clean in the source tree directory."""
assert self._build_dir
cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir)
cmd = "bazel clean"
output = cmd_exec.run_command(cmd, cmd_params)
log.debug(f"Clean output: {output}")
def execute_benchmark(self) -> None:
"""Execute the scavenging benchmark.
Raises:
BenchmarkError: if the benchmark fails to execute successfully
"""
self._validate()
self._prepare_nighthawk()
# pull in environment and set values
env = self._control.environment
output_dir = env.output_dir
images = self.get_images()
log.debug(f"Images: {images.nighthawk_benchmark_image}")
# 'TMPDIR' is required for successful operation. This is the output
# directory for all produced NightHawk artifacts
image_vars = {
'NH_DOCKER_IMAGE': images.nighthawk_binary_image,
'ENVOY_DOCKER_IMAGE_TO_TEST': images.envoy_image,
'TMPDIR': output_dir
}
log.debug(f"Using environment: {image_vars}")
for (key, value) in image_vars.items():
if key not in env.variables:
log.debug(
f"Building control environment variables: {key}={value}")
env.variables[key] = value
environment_controller = base_benchmark.BenchmarkEnvController(env)
cmd = ("bazel-bin/benchmarks/benchmarks "
"--log-cli-level=info -vvvv -k test_http_h1_small "
"benchmarks/")
cmd_params = cmd_exec.CommandParameters(cwd=self._benchmark_dir)
with environment_controller:
try:
cmd_exec.run_command(cmd, cmd_params)
except subprocess.CalledProcessError as cpe:
raise base_benchmark.BenchmarkError(
f"Unable to execute the benchmark: {cpe}")
def get_previous_commit_hash(self,
current_commit: str,
revisions: int = 2) -> str:
"""Return the specified number of commits behind the current commit hash.
Args:
current_commit: The current hash from which we are starting the search
revisions: The number of commits in the tree that we skip over, starting
from the current_commit
Returns:
a string with the discovered commit hash
Raises:
SourceTreeError if we are not able to deduce the previous commit
"""
assert current_commit
if not self.pull():
log.debug("Source pull failed. Copying source directory")
self.copy_source_directory()
log.debug(
f"Finding previous commit to current commit: [{current_commit}]")
if is_tag(current_commit):
log.info(f"Current commit \"{current_commit}\" is a tag.")
return self.get_previous_tag(current_commit)
if current_commit == 'latest':
current_commit = self.get_head_hash()
cmd = "git rev-list --no-merges --committer='GitHub <*****@*****.**>' "
cmd += "--max-count={revisions} {commit}".format(revisions=revisions,
commit=current_commit)
cmd_params = cmd_exec.CommandParameters(
cwd=self.get_source_directory())
hash_list = cmd_exec.run_command(cmd, cmd_params)
# Check whether we got an error from git
if 'unknown revision or path not in the working tree.' in hash_list:
raise SourceTreeError(hash_list)
# Reverse iterate throught the list of hashes, skipping any blank
# lines that may have trailed the original git output
for commit_hash in hash_list.split('\n')[::-1]:
if commit_hash:
log.debug(f"Returning {commit_hash} as the previous commit to "
f"{current_commit}")
return commit_hash
raise SourceTreeError(f"No commit found prior to {current_commit}")
def stage_envoy(self, strip_binary: bool) -> None:
"""Copy and optionally strip the Envoy binary.
After we compile Envoy, copy the binary into a platform directory
for inclusion in the docker image. It is unclear the intent of the
'targetplatform' parameter in the Dockerfile, so we use a static
string. Ultimately the compiled binary is staged for packaging in
the resulting image.
Args:
strip_binary: determines whether we use objcopy to strip debug
symbols from the envoy binary. If strip_binary is False, we
simply copy the binary to its destination
Callers use False for now, until we expose a control for
manipulating this parameter.
Returns:
None
"""
# Stage the envoy binary for the docker image
dir_mode = 0o755
pwd = os.getcwd()
os.chdir(self._build_dir)
if not os.path.exists('build_release_stripped'):
os.mkdir('build_release_stripped', dir_mode)
os.chdir(pwd)
cmd = "objcopy --strip-debug " if strip_binary else "cp -fv "
cmd += constants.ENVOY_BINARY_TARGET_OUTPUT_PATH
cmd += " build_release_stripped/envoy"
cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir)
cmd_exec.run_command(cmd, cmd_params)
def build_nighthawk_binaries(self) -> None:
"""Build the NightHawk client and server binaries.
This is a pre-requisite to building the nighthawk binary docker image
"""
self.prepare_nighthawk_source()
cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir)
bazel_options = self._generate_bazel_options(
proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK)
cmd = "bazel build {bazel_options} //:nighthawk".format(
bazel_options=bazel_options)
output = cmd_exec.run_command(cmd, cmd_params)
log.debug(f"Nighthawk build output: {output}")
def test_run_command_fail(mock_check_call):
"""Verify that a CalledProcessError is bubbled to the caller if the command
fails.
"""
mock_check_call.side_effect = check_call_side_effect
cmd_parameters = cmd_exec.CommandParameters(cwd='/tmp')
cmd = 'command_error'
output = ''
with pytest.raises(subprocess.CalledProcessError) as process_error:
output = cmd_exec.run_command(cmd, cmd_parameters)
assert not output
assert f"Command \'{cmd}\' returned non-zero exit status" in \
str(process_error.value)
def get_head_hash(self) -> str:
"""Retrieve the hash for the HEAD commit.
Returns:
a string containing the hash corresponding to commit at the HEAD of the
tree.
"""
self._validate()
cmd = (
"git rev-list --no-merges --committer='GitHub <*****@*****.**>' "
"--max-count=1 HEAD")
cmd_params = cmd_exec.CommandParameters(
cwd=self.get_source_directory())
return cmd_exec.run_command(cmd, cmd_params)
def list_tags(self) -> List[str]:
"""Enumerate the repository tags and return them in a list.
Returns:
a list of tags from the commits
"""
self._validate()
cmd = "git tag --list --sort v:refname"
cmd_params = cmd_exec.CommandParameters(
cwd=self.get_source_directory())
tag_output = cmd_exec.run_command(cmd, cmd_params)
tag_list = [tag.strip() for tag in tag_output.split('\n') if tag]
log.debug(f"Repository tags {tag_list}")
return tag_list
def build_nighthawk_benchmarks(self) -> None:
"""Build the NightHawk benchmarks target.
This target is required for the scavenging benchmark. It is also a pre-
requisite to building the benchmark container image
"""
self.prepare_nighthawk_source()
cmd_params = cmd_exec.CommandParameters(cwd=self._build_dir)
bazel_options = self._generate_bazel_options(
proto_source.SourceRepository.SourceIdentity.SRCID_NIGHTHAWK)
cmd = "bazel build {bazel_options} //benchmarks:benchmarks".format(
bazel_options=bazel_options)
output = cmd_exec.run_command(cmd, cmd_params)
log.debug(f"Nighthawk build output: {output}")
def get_revs_behind_parent_branch(self) -> int:
"""Get the number of commits behind the parent branch.
Determine how many commits the current branch on disk is behind the
parent branch. If we are up to date, return zero
Returns:
an integer with the number of commits the local source lags
behind the parent branch
"""
self._validate()
cmd = "git status"
output_directory = self.get_source_directory()
cmd_params = cmd_exec.CommandParameters(cwd=output_directory)
status_output = cmd_exec.run_command(cmd, cmd_params)
commit_count = 0
# Extract the commit count from lines such as:
#
# Your branch is ahead of 'origin/master' by 99 commits.
#
# or determine whether git believes we are up to date:
#
# Your branch is up to date with 'origin/master'.
ahead = re.compile(_REPO_STATUS_REGEX)
up_to_date = re.compile(r'Your branch is up to date with \'(.*)\'')
for line in status_output.split('\n'):
match = ahead.match(line)
if match:
commit_count = int(match.group(2))
log.debug(
f"Branch is {commit_count} ahead of branch {match.group(1)}"
)
break
match = up_to_date.match(line)
if match:
log.debug(f"Branch {match.group(1)} is up to date")
break
return commit_count
def _execute_docker_image_script(script: str, build_dir: str) -> None:
"""Run the specified script to build a docker image.
The docker image tags are "fixed" at "latest" for the binary container.
The benchmark image's tag can be adjusted using DOCKER_IMAGE_TAG however
this value defaults to "latest" as well. We are not currently exposing a
method to set this environment variable.
When buliding the nighthhawk components we use the most recent source by
default.
Args:
script: The shell script in the nighthawk repository that builds
the benchmark and binary docker images.
build_dir: The nighthawk source location
"""
cmd_params = cmd_exec.CommandParameters(cwd=build_dir)
output = cmd_exec.run_command(script, cmd_params)
log.debug(f"NightHawk Docker image output for {script}: {output}")
def pull(self) -> bool:
"""Retrieve the code from the repository.
Uses git to clone the source into a working directory that has read/write
permissions by salvo
Returns:
a boolean indicating whether the operation was successful
"""
self._validate()
source_name = proto_source.SourceRepository.SourceIdentity.Name(
self._source_repo.identity)
log.debug(f"Pulling [{source_name}] from origin: "
f"[{self._source_repo.source_url}]")
if not self._source_repo.source_url:
log.debug("No url specified for source. Cannot pull.")
return False
try:
if self.is_up_to_date():
return True
except subprocess.CalledProcessError:
log.info("Source likely does not exist on disk")
if not self._source_repo.source_url:
self._source_repo.source_url = self.get_origin()
# Clone into the working directory
cmd = "git clone {origin} .".format(
origin=self._source_repo.source_url)
cmd_params = cmd_exec.CommandParameters(
cwd=self.get_source_directory())
output = cmd_exec.run_command(cmd, cmd_params)
expected = 'Cloning into \'.\''
return expected in output
def get_origin(self) -> str:
"""Detect the origin url from where the code is fetched.
Returns:
A string showing the origin url for the source tree. This needed most
for remote execution where we do not ship the entire source tree
remotely. We will attempt to generate a patch between the origin
HEAD and the local HEAD. This diff is applied to the source in the
remote context.
Raises:
SourceTreeError: if we are not able to determine the origin url for
a managed source tree
"""
valid = self._validate()
log.debug(f"Valid: {valid} for object: {self}")
origin_url = self._source_repo.source_url
output_directory = self.get_source_directory()
cmd = "git remote -v"
if not origin_url:
cmd_params = cmd_exec.CommandParameters(cwd=output_directory)
output = cmd_exec.run_command(cmd, cmd_params)
for line in output.split('\n'):
match = re.match(_GIT_ORIGIN_REGEX, line)
if match:
origin_url = match.group(1)
self._source_repo.source_url = origin_url
break
if not origin_url:
raise SourceTreeError(
f"Unable to determine the origin url from {output_directory}")
return origin_url