def upload_package(self, tag: str) -> None:
hub_cmd = [
'hub', 'release', 'create', tag, '-m',
'Release %s' % tag, '-a', self.archive_tarball_path, '-a',
self.archive_checksum_path, '-t', self.git_sha1
]
hub_delete_release_cmd = ['hub', 'release', 'delete', tag]
hub_delete_tag_cmd = ['hub', 'tag', '--delete', tag]
delay_sec = 10
for attempt_index in range(1, MAX_UPLOAD_ATTEMPTS + 1):
try:
log_and_run_cmd(hub_cmd, cwd=YB_THIRDPARTY_DIR)
break
except subprocess.CalledProcessError as ex:
if attempt_index == MAX_UPLOAD_ATTEMPTS:
raise
logging.exception(
"Failed to upload release (attempt %d out of %d). Waiting for %d sec.",
attempt_index, MAX_UPLOAD_ATTEMPTS, delay_sec)
time.sleep(delay_sec)
delay_sec += 2
logging.info("Deleting release for tag %s", tag)
log_and_run_cmd_ignore_errors(hub_delete_release_cmd)
logging.info("Deleting tag %s", tag)
log_and_run_cmd_ignore_errors(hub_delete_tag_cmd)
def copy_code_to(dest_dir: str) -> None:
parent_dir = os.path.dirname(dest_dir)
assert os.path.isdir(
parent_dir), 'Directory %s does not exist' % parent_dir
assert not os.path.exists(dest_dir), 'Already exists: %s' % dest_dir
with PushDir(YB_THIRDPARTY_DIR):
current_branch_name = get_current_git_branch_name()
log_and_run_cmd(['git', 'clone', YB_THIRDPARTY_DIR, dest_dir])
with PushDir(dest_dir):
log_and_run_cmd(['git', 'checkout', current_branch_name])
rsync_code_to(dest_dir)
def rsync_code_to(rsync_dest: str) -> None:
with PushDir(YB_THIRDPARTY_DIR):
excluded_files_bytes: bytes = subprocess.check_output(
['git', 'ls-files', '--exclude-standard', '-oi', '--directory'])
assert os.path.isdir('.git')
excluded_files_path = os.path.join(os.getcwd(), '.git', 'ignores.tmp')
with open(excluded_files_path, 'wb') as excluded_files_file:
excluded_files_file.write(excluded_files_bytes)
rsync_cmd = [
'rsync', '--archive', '--verbose', '--human-readable', '--delete',
'--checksum', '--exclude', '.git',
'--exclude-from=%s' % excluded_files_path, '.', rsync_dest
]
log_and_run_cmd(rsync_cmd)
def create_package(self) -> None:
if os.path.exists(self.archive_tarball_path):
logging.info("File already exists, deleting: %s",
self.archive_tarball_path)
os.remove(self.archive_tarball_path)
# Create a symlink with a constant name so we can copy the file around and use it for
# creating artifacts for pull request builds.
archive_symlink_path = os.path.join(YB_THIRDPARTY_DIR,
'archive' + ARCHIVE_SUFFIX)
archive_checksum_symlink_path = archive_symlink_path + CHECKSUM_SUFFIX
tar_cmd = ['tar']
patterns_to_exclude = EXCLUDE_PATTERNS_RELATIVE_TO_ARCHIVE_ROOT + [
os.path.basename(file_path) for file_path in
[archive_symlink_path, archive_checksum_symlink_path]
]
for excluded_pattern in patterns_to_exclude:
tar_cmd.extend([
'--exclude',
'%s/%s' % (self.archive_dir_name, excluded_pattern)
])
for excluded_pattern in GENERAL_EXCLUDE_PATTERNS:
tar_cmd.extend(['--exclude', excluded_pattern])
tar_cmd.extend(
['-czf', self.archive_tarball_path, self.archive_dir_name])
log_and_run_cmd(tar_cmd, cwd=self.build_dir_parent)
sha256 = compute_file_sha256(self.archive_tarball_path)
with open(self.archive_checksum_path, 'w') as sha256_file:
sha256_file.write('%s %s\n' % (sha256, self.archive_tarball_name))
logging.info("Archive SHA256 checksum: %s, created checksum file: %s",
sha256, self.archive_checksum_path)
for file_path in [archive_symlink_path, archive_checksum_symlink_path]:
remove_path(file_path)
create_symlink_and_log(self.archive_tarball_path, archive_symlink_path)
create_symlink_and_log(self.archive_checksum_path,
archive_checksum_symlink_path)
def create_package(self) -> None:
if os.path.exists(self.archive_tarball_path):
logging.info("File already exists, deleting: %s",
self.archive_tarball_path)
os.remove(self.archive_tarball_path)
tar_cmd = ['tar']
for excluded_pattern in EXCLUDE_PATTERNS_RELATIVE_TO_ARCHIVE_ROOT:
tar_cmd.extend([
'--exclude',
'%s/%s' % (self.archive_dir_name, excluded_pattern)
])
for excluded_pattern in GENERAL_EXCLUDE_PATTERNS:
tar_cmd.extend(['--exclude', excluded_pattern])
tar_cmd.extend(
['-czf', self.archive_tarball_path, self.archive_dir_name])
log_and_run_cmd(tar_cmd, cwd=self.build_dir_parent)
sha256 = compute_file_sha256(self.archive_tarball_path)
with open(self.archive_sha256_path, 'w') as sha256_file:
sha256_file.write('%s %s\n' % (sha256, self.archive_tarball_name))
logging.info("Archive SHA256 checksum: %s, created checksum file: %s",
sha256, self.archive_sha256_path)
def run_ssh_cmd(ssh_args: List[str]) -> None:
log_and_run_cmd(['ssh', remote_server] + ssh_args)
def build_remotely(remote_server: str, remote_build_code_path: str) -> None:
assert remote_server is not None
assert remote_build_code_path is not None
assert remote_build_code_path.startswith('/')
def run_ssh_cmd(ssh_args: List[str]) -> None:
log_and_run_cmd(['ssh', remote_server] + ssh_args)
def run_remote_bash_script(bash_script: str) -> None:
bash_script = bash_script.strip()
log("Running script remotely: %s", bash_script)
# TODO: why exactly do we need shlex.quote here?
run_ssh_cmd(['bash', '-c', shlex.quote(bash_script)])
quoted_remote_path = shlex.quote(remote_build_code_path)
# Ensure the remote directory exists. We are not attempting to create it if it does not.
run_remote_bash_script('[[ -d %s ]]' % quoted_remote_path)
with PushDir(YB_THIRDPARTY_DIR):
local_branch_name = get_current_git_branch_name()
local_git_remotes = subprocess.check_output(
shlex.split('git remote -v')).decode('utf-8')
remote_url = '%s:%s' % (remote_server, remote_build_code_path)
preferred_remote_name = 'remote-build-%s' % remote_server
remote_name = None
for remote_line in local_git_remotes.split('\n'):
remote_line = remote_line.strip()
if not remote_line:
continue
remote_components = remote_line.split('\t')
if remote_components[1].endswith(' (push)'):
parsed_remote_url = remote_components[1][:-7].strip()
if parsed_remote_url == remote_url:
remote_name = remote_components[0]
log("Found existing remote %s for %s", remote_name,
remote_url)
break
if remote_name is None:
log_and_run_cmd(
['git', 'remote', 'add', preferred_remote_name, remote_url])
remote_name = preferred_remote_name
log("Local branch name: %s, checking it out remotely",
local_branch_name)
run_remote_bash_script(f"""
set -euo pipefail
cd {quoted_remote_path}
git reset --hard HEAD
git clean -df
git checkout master
""")
log_and_run_cmd([
'git', 'push', '--force', remote_name,
'%s:%s' % (local_branch_name, local_branch_name)
])
run_remote_bash_script(
'cd %s && git checkout %s' %
(quoted_remote_path, shlex.quote(local_branch_name)))
rsync_code_to('%s:%s' % (remote_server, remote_build_code_path))
remote_bash_script = 'cd %s && ./build_thirdparty.sh %s' % (
quoted_remote_path, shlex_join(sys.argv[1:]))
run_remote_bash_script(remote_bash_script)