def clone_commit(cirpy_dir, commit):
""" Clones the `circuitpython` repository, fetches the commit, then
checks out the repo at that ref.
"""
working_dir = pathlib.Path()
rosiepi_logger.info("Cloning repository at reference: %s", commit)
try:
git.clone("--depth", "1", "-n",
"https://github.com/sommersoft/circuitpython.git", cirpy_dir)
os.chdir(cirpy_dir)
git.fetch("origin", commit)
git.checkout(commit)
git.submodule("sync")
git.submodule("update", "--init")
except sh.ErrorReturnCode as git_err:
git_stderr = str(git_err.stderr, encoding="utf-8").strip("\n")
err_msg = [
f"Failed to retrive repository at {commit}:",
f" - {git_stderr}",
]
rosiepi_logger.warning("%s", "\n".join(err_msg))
raise RuntimeError(git_stderr) from None
finally:
os.chdir(working_dir)
def _find_branchoff_point(self, attempt_count: int = 0) -> str:
fetch_depth = 4 ** attempt_count # fetch 4, 16, 64, 256, 1024, ...
if attempt_count >= self.MAX_FETCH_ATTEMPT_COUNT: # get all commits on last try
fetch_depth = 2 ** 31 - 1 # git expects a signed 32-bit integer
if attempt_count: # skip fetching on first try
debug_echo(
f"fetching {fetch_depth} commits to find branch-off point of pull request"
)
git.fetch("origin", "--depth", fetch_depth, self.base_branch_tip)
git.fetch("origin", "--depth", fetch_depth, self.head_ref)
try: # check if both branches connect to the yet-unknown branch-off point now
process = git("merge-base", self.base_branch_tip, self.head_ref)
except sh.ErrorReturnCode as error:
output = error.stderr.decode().strip()
if (
output # output is empty when unable to find branch-off point
and "Not a valid " not in output # the error when a ref is missing
):
exit_with_sh_error(error)
if attempt_count >= self.MAX_FETCH_ATTEMPT_COUNT:
raise ActionFailure(
"Could not find branch-off point between "
f"the baseline tip {self.base_branch_tip} and current head '{self.head_ref}' "
)
return self._find_branchoff_point(attempt_count + 1)
else:
return process.stdout.decode().strip()
def add_remote(repo, name, url):
"""Add a remote to the Git repository."""
with sh.pushd(repo):
try:
git.remote("add", name, url)
except sh.ErrorReturnCode_3:
git.remote("set-url", name, url)
git.fetch(name)
def base_commit_ref(self) -> Optional[str]:
target_branch = os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
if not target_branch:
return None
head_sha = git("rev-parse", "HEAD").stdout.strip()
git.fetch(self._get_remote_url(), target_branch)
base_sha = (git("merge-base", "--all", head_sha,
"FETCH_HEAD").stdout.decode().strip())
return base_sha
def update_repo(data):
repo_name = data['repository']['name']
repo_owner = data['repository']['owner']['name']
repo_url = data['repository']['clone_url']
repo_path = get_repo_path(repo_owner, repo_name)
if not repo_path.exists():
repo_path.mkdir(parents=True)
with pushd(repo_path):
git.clone(repo_url, '.', bare=True)
else:
with pushd(repo_path):
git.fetch('origin', data['ref'])
def main():
with tempfile.TemporaryDirectory() as tmpdir:
info('Created tmp directory ' + tmpdir)
os.chdir(tmpdir)
git.clone(WIKI_DIR, 'openafs-wiki', _fg=True)
os.chdir('openafs-wiki')
git.remote('add', 'gerrit', 'ssh://gerrit.openafs.org/openafs-wiki.git')
git.fetch('gerrit', _fg=True)
git.reset('gerrit/master', '--hard', _fg=True)
update_page('devel/GerritsForMaster.mdwn', 'master')
update_page('devel/GerritsForStable.mdwn', 'openafs-stable-1_8_x')
update_page('devel/GerritsForOldStable.mdwn', 'openafs-stable-1_6_x')
try:
git.commit('-m', 'update gerrit list', _fg=True)
except ErrorReturnCode_1:
print('No changes')
else:
git.push('gerrit', 'HEAD:refs/heads/master', _fg=True)
def build_fw(board, build_ref, test_log): # pylint: disable=too-many-locals,too-many-statements
""" Builds the firware at `build_ref` for `board`. Firmware will be
output to `.fw_builds/<build_ref>/<board>/`.
:param: str board: Name of the board to build firmware for.
:param: str build_ref: The tag/commit to build firmware for.
:param: test_log: The TestController.log used for output.
"""
working_dir = os.getcwd()
cirpy_ports_dir = pathlib.Path(cirpy_dir(), "ports")
board_port_dir = None
for port in _AVAILABLE_PORTS:
port_dir = cirpy_ports_dir / port / "boards" / board
if port_dir.exists():
board_port_dir = (cirpy_ports_dir / port).resolve()
rosiepi_logger.info("Board source found: %s", board_port_dir)
break
if board_port_dir is None:
err_msg = [
f"'{board}' board not available to test. Can't build firmware.",
#"="*60,
#"Closing RosiePi"
]
raise RuntimeError("\n".join(err_msg))
build_dir = pathlib.Path.home() / ".fw_builds" / build_ref[:5] / board
os.chdir(cirpy_dir())
try:
test_log.write("Fetching {}...".format(build_ref))
git.fetch("--depth", "1", "origin", build_ref)
test_log.write("Checking out {}...".format(build_ref))
git.checkout(build_ref)
test_log.write("Syncing submodules...")
git.submodule("sync")
test_log.write("Updating submodules...")
git.submodule("update", "--init", "--depth", "1")
except sh.ErrorReturnCode as git_err:
# TODO: change to 'master'
git.checkout("-f", "rosiepi_test")
os.chdir(working_dir)
err_msg = [
"Building firmware failed:",
" - {}".format(str(git_err.stderr, encoding="utf-8").strip("\n")),
#"="*60,
#"Closing RosiePi"
]
raise RuntimeError("\n".join(err_msg)) from None
os.chdir(board_port_dir)
board_cmd = (f"make clean BOARD={board} BUILD={build_dir}",
f"make BOARD={board} BUILD={build_dir}")
test_log.write("Building firmware...")
try:
rosiepi_logger.info("Running make recipe: %s", '; '.join(board_cmd))
run_envs = {
"BASH_ENV": "/etc/profile",
}
rosiepi_logger.info("Running build clean...")
# pylint: disable=subprocess-run-check
subprocess.run(
board_cmd[0],
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
executable="/usr/bin/bash",
start_new_session=True,
env=run_envs,
)
build_dir.mkdir(mode=0o0774, parents=True)
# pylint: enable=subprocess-run-check
rosiepi_logger.info("Running firmware build...")
fw_build = subprocess.run(
board_cmd[1],
check=True,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
executable="/usr/bin/bash",
start_new_session=True,
env=run_envs,
)
result = str(fw_build.stdout, encoding="utf-8").split("\n")
success_msg = [line for line in result if "bytes" in line]
test_log.write(" - " + "\n - ".join(success_msg))
rosiepi_logger.info("Firmware built...")
except subprocess.CalledProcessError as cmd_err:
# TODO: change to 'master'
git.checkout("-f", "rosiepi_test")
os.chdir(working_dir)
err_msg = [
"Building firmware failed:",
" - {}".format(str(cmd_err.stdout, encoding="utf-8").strip("\n")),
#"="*60,
#"Closing RosiePi"
]
rosiepi_logger.warning("Firmware build failed...")
raise RuntimeError("\n".join(err_msg)) from None
finally:
# TODO: change to 'master'
git.checkout("-f", "rosiepi_test")
os.chdir(working_dir)
return build_dir
def sync_branches(self, repo, bitbucket_account_id, bitbucket_access_token,
github_account_id, github_access_token):
repo_name = repo['name']
prefixed_repo_name = self.prefix + repo_name
github_link = repo['github_link']
bitbucket_link = repo['bitbucket_link']
# Boolean: whether the repo is a new migration to GitHub
new_migration = repo['new_migration']
# Use this instead of setting the authenticated link as a new remote.
# Remote links get stored in git config
github_link_domain = github_link.split("//")[1]
authenticated_github_link = f"https://{github_account_id}:{github_access_token}@{github_link_domain}"
bitbucket_link_domain = bitbucket_link.split("//")[1]
authenticated_bitbucket_link = f"https://{bitbucket_account_id}:{bitbucket_access_token}@{bitbucket_link_domain}"
# Set remote to bitbucket
git.remote('set-url', 'origin', bitbucket_link)
self.log.debug("Syncing branches. Set origin to Bitbucket",
repo_name=repo_name,
bitbucket_link=bitbucket_link)
# Fetch branches from origin (bitbucket)
self.log.info("Fetching refs (branches) from origin",
repo_name=repo_name)
# git.fetch('origin')
git.fetch(authenticated_bitbucket_link)
self.log.debug("Fetched refs (branches) from BitBucket",
result="SUCCESS",
repo_name=repo_name)
# List remote branches
remote_branches = git.branch("-r").split("\n")
remote_branches = [
remote.lstrip().rstrip() for remote in remote_branches
if (remote and not re.match("^.*/HEAD -> .*$", remote))
]
try:
# if master exists on origin, move that to the start of the array
# pushing 'master' first makes it the default branch on github
remote_branches.remove('origin/master')
remote_branches = ['origin/master'] + remote_branches
except Exception:
# 'master' did not exist on origin
pass
success_branches = []
failed_branches = []
# Push changes to each branch individually, log error if any fails and continue to next branch
for remote in remote_branches:
[remote_name, branch_name] = remote.split('/', 1)
self.log.info("Syncing branch for repository",
repo_name=repo_name,
branch_name=branch_name)
if (remote_name == 'origin'):
# Different way to handle master branches, support prefixing.
if (branch_name == "master"):
master_branch_refspecs = []
prefix_exists = self.master_branch_prefix != ""
if (prefix_exists):
# Order is IMPORTANT, 'master' should be added before prefixed_master.
# Default branch is the first branch that is pushed to GitHub
if (new_migration):
master_branch_refspecs.append(
f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}"
)
prefixed_master_branch_name = self.master_branch_prefix + branch_name
master_branch_refspecs.append(
f"refs/remotes/origin/{branch_name}:refs/heads/{prefixed_master_branch_name}"
)
else:
master_branch_refspecs.append(
f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}"
)
for branch_refspec in master_branch_refspecs:
target_branch_name = branch_refspec.split('/')[-1]
try:
self.log.info(
"Pushing branch for repository",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=target_branch_name)
git.push(authenticated_github_link, branch_refspec)
# Success on syncing current branch
self.log.debug(
"Successfully synced branch for repository",
result="SUCCESS",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=target_branch_name)
success_branches.append(branch_name)
except ErrorReturnCode as e:
# Redact or remove the access token before logging
stderr = utils.StringUtils.redact_error(
e.stderr, github_access_token,
"<ACCESS-TOKEN>")
self.log.error(
"Failed to push changes to origin branch",
result="FAILED",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=target_branch_name,
exit_code=e.exit_code,
stderr=stderr)
failed_branches.append(branch_name)
continue # Continue to the next master_branch_refspec
continue # Continue to the next branch
branch_refspec = f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}"
try:
self.log.info("Pushing branch for repository",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=branch_name)
git.push(authenticated_github_link, branch_refspec)
# Success on syncing current branch
self.log.debug("Successfully synced branch for repository",
result="SUCCESS",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=branch_name)
success_branches.append(branch_name)
except ErrorReturnCode as e:
# Redact or remove the access token before logging
stderr = utils.StringUtils.redact_error(
e.stderr, github_access_token, "<ACCESS-TOKEN>")
self.log.error("Failed to push changes to origin branch",
result="FAILED",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=branch_name,
exit_code=e.exit_code,
stderr=stderr)
failed_branches.append(branch_name)
continue
else:
continue
all_remote_branches = [
branch_name.split('origin/')[1] for branch_name in remote_branches
]
branches_sync_success = set(all_remote_branches) == set(
success_branches)
return branches_sync_success, all_remote_branches, failed_branches
def sync_tags(self, repo, bitbucket_account_id, bitbucket_access_token,
github_account_id, github_access_token):
# Everytime, tags are fetched from remote (bitbucket) and then pushed to github
repo_name = repo['name']
prefixed_repo_name = self.prefix + repo_name
github_link = repo['github_link']
bitbucket_link = repo['bitbucket_link']
# Use this instead of setting the authenticated link as a new remote.
# Remote links get stored in git config
github_link_domain = github_link.split("//")[1]
authenticated_github_link = f"https://{github_account_id}:{github_access_token}@{github_link_domain}"
bitbucket_link_domain = bitbucket_link.split("//")[1]
authenticated_bitbucket_link = f"https://{bitbucket_account_id}:{bitbucket_access_token}@{bitbucket_link_domain}"
git.remote('set-url', 'origin', bitbucket_link)
self.log.debug("Syncing Tags. Set origin to BitBucket",
repo_name=repo_name,
bitbucket_link=bitbucket_link)
# Fetch tags from origin (bitbucket)
self.log.info("Fetching refs (tags) from origin", repo_name=repo_name)
# git.fetch('origin')
git.fetch(authenticated_bitbucket_link)
self.log.debug("Fetched refs (tags) from BitBucket",
result="SUCCESS",
repo_name=repo_name)
# List all tags
tags = git.tag().split('\n')
tags = [tag.lstrip().rstrip() for tag in tags if tag]
success_tags = []
failed_tags = []
# Set origin to github
# git.remote('set-url', 'origin', github_link)
self.log.debug("Syncing tags. Set origin to Github",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
github_link=github_link)
# Push each tag individually, log error if any fails and continue to next tag
for tag_name in tags:
self.log.info("Syncing tag for repository",
repo_name=repo_name,
tag_name=tag_name)
try:
tag_refspec = f"refs/tags/{tag_name}:refs/tags/{tag_name}"
git.push(authenticated_github_link, tag_refspec)
self.log.debug("Pushed tag for repository",
result="SUCCESS",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
tag_name=tag_name)
success_tags.append(tag_name)
except ErrorReturnCode as e:
# Redact or remove the access token before logging
stderr = utils.StringUtils.redact_error(
e.stderr, github_access_token, "<ACCESS-TOKEN>")
self.log.error("Failed to push tag to github",
result="FAILED",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
tag_name=tag_name,
exit_code=e.exit_code,
stderr=stderr)
failed_tags.append(tag_name)
continue
tags_sync_success = set(tags) == set(success_tags)
return tags_sync_success, tags, failed_tags
