def upload_package(self, tag: str) -> None: hub_cmd = [ 'hub', 'release', 'create', tag, '-m', 'Release %s' % tag, '-a', self.archive_tarball_path, '-a', self.archive_checksum_path, '-t', self.git_sha1 ] hub_delete_release_cmd = ['hub', 'release', 'delete', tag] hub_delete_tag_cmd = ['hub', 'tag', '--delete', tag] delay_sec = 10 for attempt_index in range(1, MAX_UPLOAD_ATTEMPTS + 1): try: log_and_run_cmd(hub_cmd, cwd=YB_THIRDPARTY_DIR) break except subprocess.CalledProcessError as ex: if attempt_index == MAX_UPLOAD_ATTEMPTS: raise logging.exception( "Failed to upload release (attempt %d out of %d). Waiting for %d sec.", attempt_index, MAX_UPLOAD_ATTEMPTS, delay_sec) time.sleep(delay_sec) delay_sec += 2 logging.info("Deleting release for tag %s", tag) log_and_run_cmd_ignore_errors(hub_delete_release_cmd) logging.info("Deleting tag %s", tag) log_and_run_cmd_ignore_errors(hub_delete_tag_cmd)
def copy_code_to(dest_dir: str) -> None: parent_dir = os.path.dirname(dest_dir) assert os.path.isdir( parent_dir), 'Directory %s does not exist' % parent_dir assert not os.path.exists(dest_dir), 'Already exists: %s' % dest_dir with PushDir(YB_THIRDPARTY_DIR): current_branch_name = get_current_git_branch_name() log_and_run_cmd(['git', 'clone', YB_THIRDPARTY_DIR, dest_dir]) with PushDir(dest_dir): log_and_run_cmd(['git', 'checkout', current_branch_name]) rsync_code_to(dest_dir)
def rsync_code_to(rsync_dest: str) -> None: with PushDir(YB_THIRDPARTY_DIR): excluded_files_bytes: bytes = subprocess.check_output( ['git', 'ls-files', '--exclude-standard', '-oi', '--directory']) assert os.path.isdir('.git') excluded_files_path = os.path.join(os.getcwd(), '.git', 'ignores.tmp') with open(excluded_files_path, 'wb') as excluded_files_file: excluded_files_file.write(excluded_files_bytes) rsync_cmd = [ 'rsync', '--archive', '--verbose', '--human-readable', '--delete', '--checksum', '--exclude', '.git', '--exclude-from=%s' % excluded_files_path, '.', rsync_dest ] log_and_run_cmd(rsync_cmd)
def create_package(self) -> None: if os.path.exists(self.archive_tarball_path): logging.info("File already exists, deleting: %s", self.archive_tarball_path) os.remove(self.archive_tarball_path) # Create a symlink with a constant name so we can copy the file around and use it for # creating artifacts for pull request builds. archive_symlink_path = os.path.join(YB_THIRDPARTY_DIR, 'archive' + ARCHIVE_SUFFIX) archive_checksum_symlink_path = archive_symlink_path + CHECKSUM_SUFFIX tar_cmd = ['tar'] patterns_to_exclude = EXCLUDE_PATTERNS_RELATIVE_TO_ARCHIVE_ROOT + [ os.path.basename(file_path) for file_path in [archive_symlink_path, archive_checksum_symlink_path] ] for excluded_pattern in patterns_to_exclude: tar_cmd.extend([ '--exclude', '%s/%s' % (self.archive_dir_name, excluded_pattern) ]) for excluded_pattern in GENERAL_EXCLUDE_PATTERNS: tar_cmd.extend(['--exclude', excluded_pattern]) tar_cmd.extend( ['-czf', self.archive_tarball_path, self.archive_dir_name]) log_and_run_cmd(tar_cmd, cwd=self.build_dir_parent) sha256 = compute_file_sha256(self.archive_tarball_path) with open(self.archive_checksum_path, 'w') as sha256_file: sha256_file.write('%s %s\n' % (sha256, self.archive_tarball_name)) logging.info("Archive SHA256 checksum: %s, created checksum file: %s", sha256, self.archive_checksum_path) for file_path in [archive_symlink_path, archive_checksum_symlink_path]: remove_path(file_path) create_symlink_and_log(self.archive_tarball_path, archive_symlink_path) create_symlink_and_log(self.archive_checksum_path, archive_checksum_symlink_path)
def create_package(self) -> None: if os.path.exists(self.archive_tarball_path): logging.info("File already exists, deleting: %s", self.archive_tarball_path) os.remove(self.archive_tarball_path) tar_cmd = ['tar'] for excluded_pattern in EXCLUDE_PATTERNS_RELATIVE_TO_ARCHIVE_ROOT: tar_cmd.extend([ '--exclude', '%s/%s' % (self.archive_dir_name, excluded_pattern) ]) for excluded_pattern in GENERAL_EXCLUDE_PATTERNS: tar_cmd.extend(['--exclude', excluded_pattern]) tar_cmd.extend( ['-czf', self.archive_tarball_path, self.archive_dir_name]) log_and_run_cmd(tar_cmd, cwd=self.build_dir_parent) sha256 = compute_file_sha256(self.archive_tarball_path) with open(self.archive_sha256_path, 'w') as sha256_file: sha256_file.write('%s %s\n' % (sha256, self.archive_tarball_name)) logging.info("Archive SHA256 checksum: %s, created checksum file: %s", sha256, self.archive_sha256_path)
def run_ssh_cmd(ssh_args: List[str]) -> None: log_and_run_cmd(['ssh', remote_server] + ssh_args)
def build_remotely(remote_server: str, remote_build_code_path: str) -> None: assert remote_server is not None assert remote_build_code_path is not None assert remote_build_code_path.startswith('/') def run_ssh_cmd(ssh_args: List[str]) -> None: log_and_run_cmd(['ssh', remote_server] + ssh_args) def run_remote_bash_script(bash_script: str) -> None: bash_script = bash_script.strip() log("Running script remotely: %s", bash_script) # TODO: why exactly do we need shlex.quote here? run_ssh_cmd(['bash', '-c', shlex.quote(bash_script)]) quoted_remote_path = shlex.quote(remote_build_code_path) # Ensure the remote directory exists. We are not attempting to create it if it does not. run_remote_bash_script('[[ -d %s ]]' % quoted_remote_path) with PushDir(YB_THIRDPARTY_DIR): local_branch_name = get_current_git_branch_name() local_git_remotes = subprocess.check_output( shlex.split('git remote -v')).decode('utf-8') remote_url = '%s:%s' % (remote_server, remote_build_code_path) preferred_remote_name = 'remote-build-%s' % remote_server remote_name = None for remote_line in local_git_remotes.split('\n'): remote_line = remote_line.strip() if not remote_line: continue remote_components = remote_line.split('\t') if remote_components[1].endswith(' (push)'): parsed_remote_url = remote_components[1][:-7].strip() if parsed_remote_url == remote_url: remote_name = remote_components[0] log("Found existing remote %s for %s", remote_name, remote_url) break if remote_name is None: log_and_run_cmd( ['git', 'remote', 'add', preferred_remote_name, remote_url]) remote_name = preferred_remote_name log("Local branch name: %s, checking it out remotely", local_branch_name) run_remote_bash_script(f""" set -euo pipefail cd {quoted_remote_path} git reset --hard HEAD git clean -df git checkout master """) log_and_run_cmd([ 'git', 'push', '--force', remote_name, '%s:%s' % (local_branch_name, local_branch_name) ]) run_remote_bash_script( 'cd %s && git checkout %s' % (quoted_remote_path, shlex.quote(local_branch_name))) rsync_code_to('%s:%s' % (remote_server, remote_build_code_path)) remote_bash_script = 'cd %s && ./build_thirdparty.sh %s' % ( quoted_remote_path, shlex_join(sys.argv[1:])) run_remote_bash_script(remote_bash_script)