def log_output( prefix: str, args: List[Any], disallowed_pattern: Optional[Pattern] = None) -> None: cmd_str = shlex_join(args) try: print_line_with_colored_prefix( prefix, "Running command: {} (current directory: {})".format(cmd_str, os.getcwd())) process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) assert process.stdout is not None for line in process.stdout: if disallowed_pattern and disallowed_pattern.search(line): raise RuntimeError( "Output line from command [[ {} ]] contains a disallowed pattern: {}".format( cmd_str, disallowed_pattern)) print_line_with_colored_prefix(prefix, line.decode('utf-8')) process.stdout.close() exit_code = process.wait() if exit_code: fatal("Execution failed with code: {}".format(exit_code)) except OSError as err: log("Error when trying to execute command: " + str(args)) log("PATH is: %s", os.getenv("PATH")) raise
def log_and_run_cmd_ignore_errors(args: List[Any], **kwargs: Any) -> None: args = normalize_cmd_args(args) args_str = shlex_join(args) _log_cmd_to_run(args, cwd=kwargs.get('cwd')) try: subprocess.check_call(args, **kwargs) except subprocess.CalledProcessError as ex: logging.exception("Command failed: %s (ignoring the error)", args_str, ex)
def build(self) -> BuildResult: with PushDir(self.conf_run_dir): home_dir_in_container = '/home/yugabyteci' pip_cache_dir_in_container = os.path.join(home_dir_in_container, '.cache', 'pip') readonly_checkout_in_container = '/opt/yb-build/readonly-code/yugabyte-db-thirdparty' rw_checkout_in_container = '/opt/yb-build/thirdparty/checkout' sudo_cmd = 'sudo -u yugabyteci ' # TODO: create a shell script in the checkout directory outside container with all the # right settings so we can rerun it manually easily if needed. bash_script = '; '.join([ f"set -euxo pipefail", 'mkdir -p /root/.cache/pip', 'chmod a+rX /root', 'chmod a+rX /root/.cache', 'chmod -R a+rX /root/.cache/pip', # Here, before we switch user to yugabyteci, we can do things as root if needed. 'sudo -u yugabyteci /bin/bash -c ' + shlex.quote('; '.join([ 'set -euxo pipefail', f'mkdir -p {pip_cache_dir_in_container}', f'export PIP_DOWNLOAD_CACHE={pip_cache_dir_in_container}', f"export YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX={self.archive_name_suffix}", f"export YB_BUILD_THIRDPARTY_ARGS='{self.build_thirdparty_args}'", f"cp -R {readonly_checkout_in_container} {rw_checkout_in_container}", f"cd {rw_checkout_in_container}", "./build_and_release.sh" ])) ]) container_name = f'{self.name}-{self.common_conf.timestamp_str}' docker_run_cmd_args = [ 'docker', 'run', '--name', container_name, '--cap-add=SYS_PTRACE', '--mount', ','.join([ 'type=bind', f'source={self.common_conf.checkout_dir}', f'target={readonly_checkout_in_container}', 'readonly', ]), self.docker_image, 'bash', '-c', bash_script ] self.log_with_prefix("Running command: %s", shlex_join(docker_run_cmd_args)) self.log_with_prefix("Logging to: %s", self.output_file_path) start_time_sec = time.time() with open(self.output_file_path, 'wb') as output_file: docker_run_process = subprocess.Popen(docker_run_cmd_args, stdout=output_file, stderr=subprocess.STDOUT) docker_run_process.wait() elapsed_time_sec = time.time() - start_time_sec self.log_with_prefix("Return code: %d, elapsed time: %.1f sec", docker_run_process.returncode, elapsed_time_sec) return BuildResult()
def build_remotely(remote_server: str, remote_build_code_path: str) -> None: assert remote_server is not None assert remote_build_code_path is not None assert remote_build_code_path.startswith('/') def run_ssh_cmd(ssh_args: List[str]) -> None: log_and_run_cmd(['ssh', remote_server] + ssh_args) def run_remote_bash_script(bash_script: str) -> None: bash_script = bash_script.strip() log("Running script remotely: %s", bash_script) # TODO: why exactly do we need shlex.quote here? run_ssh_cmd(['bash', '-c', shlex.quote(bash_script)]) quoted_remote_path = shlex.quote(remote_build_code_path) # Ensure the remote directory exists. We are not attempting to create it if it does not. run_remote_bash_script('[[ -d %s ]]' % quoted_remote_path) with PushDir(YB_THIRDPARTY_DIR): local_branch_name = get_current_git_branch_name() local_git_remotes = subprocess.check_output( shlex.split('git remote -v')).decode('utf-8') remote_url = '%s:%s' % (remote_server, remote_build_code_path) preferred_remote_name = 'remote-build-%s' % remote_server remote_name = None for remote_line in local_git_remotes.split('\n'): remote_line = remote_line.strip() if not remote_line: continue remote_components = remote_line.split('\t') if remote_components[1].endswith(' (push)'): parsed_remote_url = remote_components[1][:-7].strip() if parsed_remote_url == remote_url: remote_name = remote_components[0] log("Found existing remote %s for %s", remote_name, remote_url) break if remote_name is None: log_and_run_cmd( ['git', 'remote', 'add', preferred_remote_name, remote_url]) remote_name = preferred_remote_name log("Local branch name: %s, checking it out remotely", local_branch_name) run_remote_bash_script(f""" set -euo pipefail cd {quoted_remote_path} git reset --hard HEAD git clean -df git checkout master """) log_and_run_cmd([ 'git', 'push', '--force', remote_name, '%s:%s' % (local_branch_name, local_branch_name) ]) run_remote_bash_script( 'cd %s && git checkout %s' % (quoted_remote_path, shlex.quote(local_branch_name))) rsync_code_to('%s:%s' % (remote_server, remote_build_code_path)) remote_bash_script = 'cd %s && ./build_thirdparty.sh %s' % ( quoted_remote_path, shlex_join(sys.argv[1:])) run_remote_bash_script(remote_bash_script)
def _log_cmd_to_run(args: List[str]) -> None: log("Running command: %s (current directory: %s)", shlex_join(args), os.getcwd())
def _log_cmd_to_run(args: List[str], cwd: Optional[Any]) -> None: cwd = cwd or os.getcwd() log("Running command: %s (in directory: %s)", shlex_join(args), cwd)
def ensure_file_downloaded( self, url: str, file_path: str, enable_using_alternative_url: bool, expected_checksum: Optional[str] = None, verify_checksum: bool = True) -> None: log(f"Ensuring {url} is downloaded to path {file_path}") file_name = os.path.basename(file_path) mkdir_if_missing(self.download_dir) if os.path.exists(file_path) and verify_checksum: # We check the filename against our checksum map only if the file exists. This is done # so that we would still download the file even if we don't know the checksum, making it # easier to add new third-party dependencies. if expected_checksum is None: expected_checksum = self.get_expected_checksum_and_maybe_add_to_file( file_name, downloaded_path=file_path) if self.verify_checksum(file_path, expected_checksum): log("No need to re-download %s: checksum already correct", file_name) return log("File %s already exists but has wrong checksum, removing", file_path) remove_path(file_path) log("Fetching %s from %s", file_name, url) download_successful = False alternative_url = ALTERNATIVE_URL_PREFIX + file_name total_attempts = 0 url_candidates = [url] if enable_using_alternative_url: url_candidates += [alternative_url] for effective_url in url_candidates: if effective_url == alternative_url: log("Switching to alternative download URL %s after %d attempts", alternative_url, total_attempts) sleep_time_sec = INITIAL_DOWNLOAD_RETRY_SLEEP_TIME_SEC for attempt_index in range(1, MAX_FETCH_ATTEMPTS + 1): try: total_attempts += 1 curl_cmd_line = [ self.curl_path, '-o', file_path, '-L', # follow redirects '--silent', '--show-error', '--location', effective_url] log("Running command: %s", shlex_join(curl_cmd_line)) subprocess.check_call(curl_cmd_line) download_successful = True break except subprocess.CalledProcessError as ex: log("Error downloading %s (attempt %d for this URL, total attempts %d): %s", self.curl_path, attempt_index, total_attempts, str(ex)) if attempt_index == MAX_FETCH_ATTEMPTS and effective_url == alternative_url: log("Giving up after %d attempts", MAX_FETCH_ATTEMPTS) raise ex log("Will retry after %.1f seconds", sleep_time_sec) time.sleep(sleep_time_sec) sleep_time_sec += DOWNLOAD_RETRY_SLEEP_INCREASE_SEC if download_successful: break if not os.path.exists(file_path): fatal("Downloaded '%s' but but unable to find '%s'", url, file_path) if verify_checksum: if expected_checksum is None: expected_checksum = self.get_expected_checksum_and_maybe_add_to_file( file_name, downloaded_path=file_path) if not self.verify_checksum(file_path, expected_checksum): fatal("File '%s' has wrong checksum after downloading from '%s'. " "Has %s, but expected: %s", file_path, url, compute_file_sha256(file_path), expected_checksum)