def log_output(
prefix: str,
args: List[Any],
disallowed_pattern: Optional[Pattern] = None) -> None:
cmd_str = shlex_join(args)
try:
print_line_with_colored_prefix(
prefix, "Running command: {} (current directory: {})".format(cmd_str, os.getcwd()))
process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
assert process.stdout is not None
for line in process.stdout:
if disallowed_pattern and disallowed_pattern.search(line):
raise RuntimeError(
"Output line from command [[ {} ]] contains a disallowed pattern: {}".format(
cmd_str, disallowed_pattern))
print_line_with_colored_prefix(prefix, line.decode('utf-8'))
process.stdout.close()
exit_code = process.wait()
if exit_code:
fatal("Execution failed with code: {}".format(exit_code))
except OSError as err:
log("Error when trying to execute command: " + str(args))
log("PATH is: %s", os.getenv("PATH"))
raise
def log_and_run_cmd_ignore_errors(args: List[Any], **kwargs: Any) -> None:
args = normalize_cmd_args(args)
args_str = shlex_join(args)
_log_cmd_to_run(args, cwd=kwargs.get('cwd'))
try:
subprocess.check_call(args, **kwargs)
except subprocess.CalledProcessError as ex:
logging.exception("Command failed: %s (ignoring the error)", args_str,
ex)
def build(self) -> BuildResult:
with PushDir(self.conf_run_dir):
home_dir_in_container = '/home/yugabyteci'
pip_cache_dir_in_container = os.path.join(home_dir_in_container,
'.cache', 'pip')
readonly_checkout_in_container = '/opt/yb-build/readonly-code/yugabyte-db-thirdparty'
rw_checkout_in_container = '/opt/yb-build/thirdparty/checkout'
sudo_cmd = 'sudo -u yugabyteci '
# TODO: create a shell script in the checkout directory outside container with all the
# right settings so we can rerun it manually easily if needed.
bash_script = '; '.join([
f"set -euxo pipefail",
'mkdir -p /root/.cache/pip',
'chmod a+rX /root',
'chmod a+rX /root/.cache',
'chmod -R a+rX /root/.cache/pip',
# Here, before we switch user to yugabyteci, we can do things as root if needed.
'sudo -u yugabyteci /bin/bash -c ' + shlex.quote('; '.join([
'set -euxo pipefail',
f'mkdir -p {pip_cache_dir_in_container}',
f'export PIP_DOWNLOAD_CACHE={pip_cache_dir_in_container}',
f"export YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX={self.archive_name_suffix}",
f"export YB_BUILD_THIRDPARTY_ARGS='{self.build_thirdparty_args}'",
f"cp -R {readonly_checkout_in_container} {rw_checkout_in_container}",
f"cd {rw_checkout_in_container}", "./build_and_release.sh"
]))
])
container_name = f'{self.name}-{self.common_conf.timestamp_str}'
docker_run_cmd_args = [
'docker', 'run', '--name', container_name,
'--cap-add=SYS_PTRACE', '--mount', ','.join([
'type=bind',
f'source={self.common_conf.checkout_dir}',
f'target={readonly_checkout_in_container}',
'readonly',
]), self.docker_image, 'bash', '-c', bash_script
]
self.log_with_prefix("Running command: %s",
shlex_join(docker_run_cmd_args))
self.log_with_prefix("Logging to: %s", self.output_file_path)
start_time_sec = time.time()
with open(self.output_file_path, 'wb') as output_file:
docker_run_process = subprocess.Popen(docker_run_cmd_args,
stdout=output_file,
stderr=subprocess.STDOUT)
docker_run_process.wait()
elapsed_time_sec = time.time() - start_time_sec
self.log_with_prefix("Return code: %d, elapsed time: %.1f sec",
docker_run_process.returncode,
elapsed_time_sec)
return BuildResult()
def build_remotely(remote_server: str, remote_build_code_path: str) -> None:
assert remote_server is not None
assert remote_build_code_path is not None
assert remote_build_code_path.startswith('/')
def run_ssh_cmd(ssh_args: List[str]) -> None:
log_and_run_cmd(['ssh', remote_server] + ssh_args)
def run_remote_bash_script(bash_script: str) -> None:
bash_script = bash_script.strip()
log("Running script remotely: %s", bash_script)
# TODO: why exactly do we need shlex.quote here?
run_ssh_cmd(['bash', '-c', shlex.quote(bash_script)])
quoted_remote_path = shlex.quote(remote_build_code_path)
# Ensure the remote directory exists. We are not attempting to create it if it does not.
run_remote_bash_script('[[ -d %s ]]' % quoted_remote_path)
with PushDir(YB_THIRDPARTY_DIR):
local_branch_name = get_current_git_branch_name()
local_git_remotes = subprocess.check_output(
shlex.split('git remote -v')).decode('utf-8')
remote_url = '%s:%s' % (remote_server, remote_build_code_path)
preferred_remote_name = 'remote-build-%s' % remote_server
remote_name = None
for remote_line in local_git_remotes.split('\n'):
remote_line = remote_line.strip()
if not remote_line:
continue
remote_components = remote_line.split('\t')
if remote_components[1].endswith(' (push)'):
parsed_remote_url = remote_components[1][:-7].strip()
if parsed_remote_url == remote_url:
remote_name = remote_components[0]
log("Found existing remote %s for %s", remote_name,
remote_url)
break
if remote_name is None:
log_and_run_cmd(
['git', 'remote', 'add', preferred_remote_name, remote_url])
remote_name = preferred_remote_name
log("Local branch name: %s, checking it out remotely",
local_branch_name)
run_remote_bash_script(f"""
set -euo pipefail
cd {quoted_remote_path}
git reset --hard HEAD
git clean -df
git checkout master
""")
log_and_run_cmd([
'git', 'push', '--force', remote_name,
'%s:%s' % (local_branch_name, local_branch_name)
])
run_remote_bash_script(
'cd %s && git checkout %s' %
(quoted_remote_path, shlex.quote(local_branch_name)))
rsync_code_to('%s:%s' % (remote_server, remote_build_code_path))
remote_bash_script = 'cd %s && ./build_thirdparty.sh %s' % (
quoted_remote_path, shlex_join(sys.argv[1:]))
run_remote_bash_script(remote_bash_script)
def _log_cmd_to_run(args: List[str]) -> None:
log("Running command: %s (current directory: %s)", shlex_join(args),
os.getcwd())
def _log_cmd_to_run(args: List[str], cwd: Optional[Any]) -> None:
cwd = cwd or os.getcwd()
log("Running command: %s (in directory: %s)", shlex_join(args), cwd)
def ensure_file_downloaded(
self,
url: str,
file_path: str,
enable_using_alternative_url: bool,
expected_checksum: Optional[str] = None,
verify_checksum: bool = True) -> None:
log(f"Ensuring {url} is downloaded to path {file_path}")
file_name = os.path.basename(file_path)
mkdir_if_missing(self.download_dir)
if os.path.exists(file_path) and verify_checksum:
# We check the filename against our checksum map only if the file exists. This is done
# so that we would still download the file even if we don't know the checksum, making it
# easier to add new third-party dependencies.
if expected_checksum is None:
expected_checksum = self.get_expected_checksum_and_maybe_add_to_file(
file_name, downloaded_path=file_path)
if self.verify_checksum(file_path, expected_checksum):
log("No need to re-download %s: checksum already correct", file_name)
return
log("File %s already exists but has wrong checksum, removing", file_path)
remove_path(file_path)
log("Fetching %s from %s", file_name, url)
download_successful = False
alternative_url = ALTERNATIVE_URL_PREFIX + file_name
total_attempts = 0
url_candidates = [url]
if enable_using_alternative_url:
url_candidates += [alternative_url]
for effective_url in url_candidates:
if effective_url == alternative_url:
log("Switching to alternative download URL %s after %d attempts",
alternative_url, total_attempts)
sleep_time_sec = INITIAL_DOWNLOAD_RETRY_SLEEP_TIME_SEC
for attempt_index in range(1, MAX_FETCH_ATTEMPTS + 1):
try:
total_attempts += 1
curl_cmd_line = [
self.curl_path,
'-o',
file_path,
'-L', # follow redirects
'--silent',
'--show-error',
'--location',
effective_url]
log("Running command: %s", shlex_join(curl_cmd_line))
subprocess.check_call(curl_cmd_line)
download_successful = True
break
except subprocess.CalledProcessError as ex:
log("Error downloading %s (attempt %d for this URL, total attempts %d): %s",
self.curl_path, attempt_index, total_attempts, str(ex))
if attempt_index == MAX_FETCH_ATTEMPTS and effective_url == alternative_url:
log("Giving up after %d attempts", MAX_FETCH_ATTEMPTS)
raise ex
log("Will retry after %.1f seconds", sleep_time_sec)
time.sleep(sleep_time_sec)
sleep_time_sec += DOWNLOAD_RETRY_SLEEP_INCREASE_SEC
if download_successful:
break
if not os.path.exists(file_path):
fatal("Downloaded '%s' but but unable to find '%s'", url, file_path)
if verify_checksum:
if expected_checksum is None:
expected_checksum = self.get_expected_checksum_and_maybe_add_to_file(
file_name, downloaded_path=file_path)
if not self.verify_checksum(file_path, expected_checksum):
fatal("File '%s' has wrong checksum after downloading from '%s'. "
"Has %s, but expected: %s",
file_path,
url,
compute_file_sha256(file_path),
expected_checksum)
