def clone_commit(cirpy_dir, commit): """ Clones the `circuitpython` repository, fetches the commit, then checks out the repo at that ref. """ working_dir = pathlib.Path() rosiepi_logger.info("Cloning repository at reference: %s", commit) try: git.clone("--depth", "1", "-n", "https://github.com/sommersoft/circuitpython.git", cirpy_dir) os.chdir(cirpy_dir) git.fetch("origin", commit) git.checkout(commit) git.submodule("sync") git.submodule("update", "--init") except sh.ErrorReturnCode as git_err: git_stderr = str(git_err.stderr, encoding="utf-8").strip("\n") err_msg = [ f"Failed to retrive repository at {commit}:", f" - {git_stderr}", ] rosiepi_logger.warning("%s", "\n".join(err_msg)) raise RuntimeError(git_stderr) from None finally: os.chdir(working_dir)
def _find_branchoff_point(self, attempt_count: int = 0) -> str: fetch_depth = 4 ** attempt_count # fetch 4, 16, 64, 256, 1024, ... if attempt_count >= self.MAX_FETCH_ATTEMPT_COUNT: # get all commits on last try fetch_depth = 2 ** 31 - 1 # git expects a signed 32-bit integer if attempt_count: # skip fetching on first try debug_echo( f"fetching {fetch_depth} commits to find branch-off point of pull request" ) git.fetch("origin", "--depth", fetch_depth, self.base_branch_tip) git.fetch("origin", "--depth", fetch_depth, self.head_ref) try: # check if both branches connect to the yet-unknown branch-off point now process = git("merge-base", self.base_branch_tip, self.head_ref) except sh.ErrorReturnCode as error: output = error.stderr.decode().strip() if ( output # output is empty when unable to find branch-off point and "Not a valid " not in output # the error when a ref is missing ): exit_with_sh_error(error) if attempt_count >= self.MAX_FETCH_ATTEMPT_COUNT: raise ActionFailure( "Could not find branch-off point between " f"the baseline tip {self.base_branch_tip} and current head '{self.head_ref}' " ) return self._find_branchoff_point(attempt_count + 1) else: return process.stdout.decode().strip()
def add_remote(repo, name, url): """Add a remote to the Git repository.""" with sh.pushd(repo): try: git.remote("add", name, url) except sh.ErrorReturnCode_3: git.remote("set-url", name, url) git.fetch(name)
def base_commit_ref(self) -> Optional[str]: target_branch = os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME") if not target_branch: return None head_sha = git("rev-parse", "HEAD").stdout.strip() git.fetch(self._get_remote_url(), target_branch) base_sha = (git("merge-base", "--all", head_sha, "FETCH_HEAD").stdout.decode().strip()) return base_sha
def update_repo(data): repo_name = data['repository']['name'] repo_owner = data['repository']['owner']['name'] repo_url = data['repository']['clone_url'] repo_path = get_repo_path(repo_owner, repo_name) if not repo_path.exists(): repo_path.mkdir(parents=True) with pushd(repo_path): git.clone(repo_url, '.', bare=True) else: with pushd(repo_path): git.fetch('origin', data['ref'])
def main(): with tempfile.TemporaryDirectory() as tmpdir: info('Created tmp directory ' + tmpdir) os.chdir(tmpdir) git.clone(WIKI_DIR, 'openafs-wiki', _fg=True) os.chdir('openafs-wiki') git.remote('add', 'gerrit', 'ssh://gerrit.openafs.org/openafs-wiki.git') git.fetch('gerrit', _fg=True) git.reset('gerrit/master', '--hard', _fg=True) update_page('devel/GerritsForMaster.mdwn', 'master') update_page('devel/GerritsForStable.mdwn', 'openafs-stable-1_8_x') update_page('devel/GerritsForOldStable.mdwn', 'openafs-stable-1_6_x') try: git.commit('-m', 'update gerrit list', _fg=True) except ErrorReturnCode_1: print('No changes') else: git.push('gerrit', 'HEAD:refs/heads/master', _fg=True)
def build_fw(board, build_ref, test_log): # pylint: disable=too-many-locals,too-many-statements """ Builds the firware at `build_ref` for `board`. Firmware will be output to `.fw_builds/<build_ref>/<board>/`. :param: str board: Name of the board to build firmware for. :param: str build_ref: The tag/commit to build firmware for. :param: test_log: The TestController.log used for output. """ working_dir = os.getcwd() cirpy_ports_dir = pathlib.Path(cirpy_dir(), "ports") board_port_dir = None for port in _AVAILABLE_PORTS: port_dir = cirpy_ports_dir / port / "boards" / board if port_dir.exists(): board_port_dir = (cirpy_ports_dir / port).resolve() rosiepi_logger.info("Board source found: %s", board_port_dir) break if board_port_dir is None: err_msg = [ f"'{board}' board not available to test. Can't build firmware.", #"="*60, #"Closing RosiePi" ] raise RuntimeError("\n".join(err_msg)) build_dir = pathlib.Path.home() / ".fw_builds" / build_ref[:5] / board os.chdir(cirpy_dir()) try: test_log.write("Fetching {}...".format(build_ref)) git.fetch("--depth", "1", "origin", build_ref) test_log.write("Checking out {}...".format(build_ref)) git.checkout(build_ref) test_log.write("Syncing submodules...") git.submodule("sync") test_log.write("Updating submodules...") git.submodule("update", "--init", "--depth", "1") except sh.ErrorReturnCode as git_err: # TODO: change to 'master' git.checkout("-f", "rosiepi_test") os.chdir(working_dir) err_msg = [ "Building firmware failed:", " - {}".format(str(git_err.stderr, encoding="utf-8").strip("\n")), #"="*60, #"Closing RosiePi" ] raise RuntimeError("\n".join(err_msg)) from None os.chdir(board_port_dir) board_cmd = (f"make clean BOARD={board} BUILD={build_dir}", f"make BOARD={board} BUILD={build_dir}") test_log.write("Building firmware...") try: rosiepi_logger.info("Running make recipe: %s", '; '.join(board_cmd)) run_envs = { "BASH_ENV": "/etc/profile", } rosiepi_logger.info("Running build clean...") # pylint: disable=subprocess-run-check subprocess.run( board_cmd[0], shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, executable="/usr/bin/bash", start_new_session=True, env=run_envs, ) build_dir.mkdir(mode=0o0774, parents=True) # pylint: enable=subprocess-run-check rosiepi_logger.info("Running firmware build...") fw_build = subprocess.run( board_cmd[1], check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, executable="/usr/bin/bash", start_new_session=True, env=run_envs, ) result = str(fw_build.stdout, encoding="utf-8").split("\n") success_msg = [line for line in result if "bytes" in line] test_log.write(" - " + "\n - ".join(success_msg)) rosiepi_logger.info("Firmware built...") except subprocess.CalledProcessError as cmd_err: # TODO: change to 'master' git.checkout("-f", "rosiepi_test") os.chdir(working_dir) err_msg = [ "Building firmware failed:", " - {}".format(str(cmd_err.stdout, encoding="utf-8").strip("\n")), #"="*60, #"Closing RosiePi" ] rosiepi_logger.warning("Firmware build failed...") raise RuntimeError("\n".join(err_msg)) from None finally: # TODO: change to 'master' git.checkout("-f", "rosiepi_test") os.chdir(working_dir) return build_dir
def sync_branches(self, repo, bitbucket_account_id, bitbucket_access_token, github_account_id, github_access_token): repo_name = repo['name'] prefixed_repo_name = self.prefix + repo_name github_link = repo['github_link'] bitbucket_link = repo['bitbucket_link'] # Boolean: whether the repo is a new migration to GitHub new_migration = repo['new_migration'] # Use this instead of setting the authenticated link as a new remote. # Remote links get stored in git config github_link_domain = github_link.split("//")[1] authenticated_github_link = f"https://{github_account_id}:{github_access_token}@{github_link_domain}" bitbucket_link_domain = bitbucket_link.split("//")[1] authenticated_bitbucket_link = f"https://{bitbucket_account_id}:{bitbucket_access_token}@{bitbucket_link_domain}" # Set remote to bitbucket git.remote('set-url', 'origin', bitbucket_link) self.log.debug("Syncing branches. Set origin to Bitbucket", repo_name=repo_name, bitbucket_link=bitbucket_link) # Fetch branches from origin (bitbucket) self.log.info("Fetching refs (branches) from origin", repo_name=repo_name) # git.fetch('origin') git.fetch(authenticated_bitbucket_link) self.log.debug("Fetched refs (branches) from BitBucket", result="SUCCESS", repo_name=repo_name) # List remote branches remote_branches = git.branch("-r").split("\n") remote_branches = [ remote.lstrip().rstrip() for remote in remote_branches if (remote and not re.match("^.*/HEAD -> .*$", remote)) ] try: # if master exists on origin, move that to the start of the array # pushing 'master' first makes it the default branch on github remote_branches.remove('origin/master') remote_branches = ['origin/master'] + remote_branches except Exception: # 'master' did not exist on origin pass success_branches = [] failed_branches = [] # Push changes to each branch individually, log error if any fails and continue to next branch for remote in remote_branches: [remote_name, branch_name] = remote.split('/', 1) self.log.info("Syncing branch for repository", repo_name=repo_name, branch_name=branch_name) if (remote_name == 'origin'): # Different way to handle master branches, support prefixing. if (branch_name == "master"): master_branch_refspecs = [] prefix_exists = self.master_branch_prefix != "" if (prefix_exists): # Order is IMPORTANT, 'master' should be added before prefixed_master. # Default branch is the first branch that is pushed to GitHub if (new_migration): master_branch_refspecs.append( f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}" ) prefixed_master_branch_name = self.master_branch_prefix + branch_name master_branch_refspecs.append( f"refs/remotes/origin/{branch_name}:refs/heads/{prefixed_master_branch_name}" ) else: master_branch_refspecs.append( f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}" ) for branch_refspec in master_branch_refspecs: target_branch_name = branch_refspec.split('/')[-1] try: self.log.info( "Pushing branch for repository", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=target_branch_name) git.push(authenticated_github_link, branch_refspec) # Success on syncing current branch self.log.debug( "Successfully synced branch for repository", result="SUCCESS", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=target_branch_name) success_branches.append(branch_name) except ErrorReturnCode as e: # Redact or remove the access token before logging stderr = utils.StringUtils.redact_error( e.stderr, github_access_token, "<ACCESS-TOKEN>") self.log.error( "Failed to push changes to origin branch", result="FAILED", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=target_branch_name, exit_code=e.exit_code, stderr=stderr) failed_branches.append(branch_name) continue # Continue to the next master_branch_refspec continue # Continue to the next branch branch_refspec = f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}" try: self.log.info("Pushing branch for repository", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=branch_name) git.push(authenticated_github_link, branch_refspec) # Success on syncing current branch self.log.debug("Successfully synced branch for repository", result="SUCCESS", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=branch_name) success_branches.append(branch_name) except ErrorReturnCode as e: # Redact or remove the access token before logging stderr = utils.StringUtils.redact_error( e.stderr, github_access_token, "<ACCESS-TOKEN>") self.log.error("Failed to push changes to origin branch", result="FAILED", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=branch_name, exit_code=e.exit_code, stderr=stderr) failed_branches.append(branch_name) continue else: continue all_remote_branches = [ branch_name.split('origin/')[1] for branch_name in remote_branches ] branches_sync_success = set(all_remote_branches) == set( success_branches) return branches_sync_success, all_remote_branches, failed_branches
def sync_tags(self, repo, bitbucket_account_id, bitbucket_access_token, github_account_id, github_access_token): # Everytime, tags are fetched from remote (bitbucket) and then pushed to github repo_name = repo['name'] prefixed_repo_name = self.prefix + repo_name github_link = repo['github_link'] bitbucket_link = repo['bitbucket_link'] # Use this instead of setting the authenticated link as a new remote. # Remote links get stored in git config github_link_domain = github_link.split("//")[1] authenticated_github_link = f"https://{github_account_id}:{github_access_token}@{github_link_domain}" bitbucket_link_domain = bitbucket_link.split("//")[1] authenticated_bitbucket_link = f"https://{bitbucket_account_id}:{bitbucket_access_token}@{bitbucket_link_domain}" git.remote('set-url', 'origin', bitbucket_link) self.log.debug("Syncing Tags. Set origin to BitBucket", repo_name=repo_name, bitbucket_link=bitbucket_link) # Fetch tags from origin (bitbucket) self.log.info("Fetching refs (tags) from origin", repo_name=repo_name) # git.fetch('origin') git.fetch(authenticated_bitbucket_link) self.log.debug("Fetched refs (tags) from BitBucket", result="SUCCESS", repo_name=repo_name) # List all tags tags = git.tag().split('\n') tags = [tag.lstrip().rstrip() for tag in tags if tag] success_tags = [] failed_tags = [] # Set origin to github # git.remote('set-url', 'origin', github_link) self.log.debug("Syncing tags. Set origin to Github", repo_name=prefixed_repo_name, repo_prefix=self.prefix, github_link=github_link) # Push each tag individually, log error if any fails and continue to next tag for tag_name in tags: self.log.info("Syncing tag for repository", repo_name=repo_name, tag_name=tag_name) try: tag_refspec = f"refs/tags/{tag_name}:refs/tags/{tag_name}" git.push(authenticated_github_link, tag_refspec) self.log.debug("Pushed tag for repository", result="SUCCESS", repo_name=prefixed_repo_name, repo_prefix=self.prefix, tag_name=tag_name) success_tags.append(tag_name) except ErrorReturnCode as e: # Redact or remove the access token before logging stderr = utils.StringUtils.redact_error( e.stderr, github_access_token, "<ACCESS-TOKEN>") self.log.error("Failed to push tag to github", result="FAILED", repo_name=prefixed_repo_name, repo_prefix=self.prefix, tag_name=tag_name, exit_code=e.exit_code, stderr=stderr) failed_tags.append(tag_name) continue tags_sync_success = set(tags) == set(success_tags) return tags_sync_success, tags, failed_tags