def download_impl(self): """Download the selected version.""" nonlocal target_dir, git directory = cls.SRC_FILE if target_dir is None else target_dir Git(self.repository, directory) with local.cwd(directory): git("checkout", self.version)
def install_uchroot(_): """Installer for erlent (contains uchroot).""" builddir = local.path(str(CFG["build_dir"].value)) with local.cwd(builddir): erlent_src = local.path('erlent') erlent_git = erlent_src / '.git' erlent_repo = str(CFG['uchroot']['repo']) erlent_build = erlent_src / 'build' if not erlent_git.exists(): git("clone", erlent_repo) else: with local.cwd(erlent_src): git("pull", "--rebase") erlent_build.mkdir() with local.cwd(erlent_build): cmake("../") make() os.environ["PATH"] = os.path.pathsep.join( [erlent_build, os.environ["PATH"]]) local.env.update(PATH=os.environ["PATH"]) if not find_package("uchroot"): LOG.error('uchroot not found, after updating PATH to %s', os.environ['PATH']) sys.exit(-1) env = CFG['env'].value if 'PATH' not in env: env['PATH'] = [] env['PATH'].append(str(erlent_build))
def test_vara_test_repo_lib_checkout(self) -> None: """Test if the repositories are checked out at the specified revision.""" self.elementalist.version( f"{self.bb_result_report_path}/" f"TwoLibsOneProjectInteractionDiscreteLibsSingleProject" f"-cpp_projects@{self.revision}", version=self.revision) # Are repositories checked out at correct commit hash? with local.cwd(self.bb_result_lib_path / "Elementalist"): self.assertEqual(self.revision[:7], git('rev-parse', '--short', 'HEAD').rstrip()) with local.cwd(self.bb_result_lib_path / "fire_lib"): self.assertEqual("ead5e00", git('rev-parse', '--short', 'HEAD').rstrip()) with local.cwd(self.bb_result_lib_path / "water_lib"): self.assertEqual("58ec513", git('rev-parse', '--short', 'HEAD').rstrip()) with local.cwd(self.bb_result_lib_path / "earth_lib"): self.assertEqual("1db6fbe", git('rev-parse', '--short', 'HEAD').rstrip())
def calc_repo_loc(repo: pygit2.Repository, rev_range: str) -> int: """ Calculate the LOC for a project at its HEAD. Args: repo: the repository to calculate the LOC for Returns: the number of lines in source-code files """ project_path = repo.path[:-5] churn_config = ChurnConfig.create_c_style_languages_config() file_pattern = re.compile("|".join( churn_config.get_extensions_repr(r"^.*\.", r"$"))) loc: int = 0 with local.cwd(project_path): files = git( "ls-tree", "-r", "--name-only", rev_range, ).splitlines() for file in files: if file_pattern.match(file): lines = git("show", f"{rev_range}:{file}").splitlines() loc += len([line for line in lines if line]) return loc
def checkout_new_branch(repo_folder: Path, branch: str, remote_branch: tp.Optional[str] = None) -> None: """Checks out a new branch in the repository.""" args = ["checkout", "-b", branch] if remote_branch is not None: args.append(remote_branch) git("-C", repo_folder.absolute(), args)
def copy_to_env(self, path: Path) -> None: with self.__lock: bb_tmp = str(path / "benchbuild/tmp") settings.bb_cfg()["tmp_dir"] = bb_tmp base.CFG["tmp_dir"] = bb_tmp git( "clone", "--dissociate", "--recurse-submodules", "--reference", self.__local, self.__remote, f"{bb_tmp}/{self.__repo_name}" )
def fetch_remote(remote: tp.Optional[str] = None, repo_folder: tp.Optional[Path] = None, extra_args: tp.Optional[tp.List[str]] = None) -> None: """Fetches the new changes from the remote.""" args = ["fetch"] if extra_args: args += extra_args if remote: args.append(remote) git("-C", repo_folder, args)
def compile(self): self.download() with local.cwd(self.src_file): git("fetch", "origin", "pull/17/head:clang") git("checkout", "clang") run.run(make["config"]) clang = compiler.cc(self) clang_cxx = compiler.cxx(self) run.run(make["CC=" + str(clang), "CXX=" + str(clang_cxx), "clean", "lsh", "sh"])
def compile(self) -> None: coreutils_source = local.path(self.source_of_primary) compiler = bb.compiler.cc(self) with local.cwd(coreutils_source): git("submodule", "init") git("submodule", "update") with local.env(CC=str(compiler)): bb.watch(local["./bootstrap"])() bb.watch(local["./configure"])("--disable-gcc-warnings") bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) verify_binaries(self)
def get_submodule_head(project_name: str, submodule_name: str, commit: FullCommitHash) -> FullCommitHash: """ Retrieve the checked out commit for a submodule of a project. Args: project_name: name of the project submodule_name: name of the submodule commit: commit of the project's main repo Returns: checked out commit of the submodule """ if submodule_name == get_primary_project_source(project_name).local: return commit main_repo = get_local_project_git_path(project_name) submodule_status = git(__get_git_path_arg(main_repo), "ls-tree", commit) commit_pattern = re.compile(r"[0-9]* commit ([0-9abcdef]*)\t" + submodule_name) match = commit_pattern.search(submodule_status) if match: return FullCommitHash(match.group(1)) raise AssertionError(f"Unknown submodule {submodule_name}")
def __clone_needed__(repository: str, directory: str) -> bool: """ Do we need to create a fresh clone of the given repository. Args: repository: the repository we want to clone. directory: the directory we expect the clone to live. Returns: True, if the clone is required. False, if the directory is a valid clone. """ from benchbuild.utils.cmd import git, rm git_dir = local.path(directory) / '.git' if not git_dir.exists(): return True requires_clone = True with local.cwd(directory): repo_origin_url = git('config', '--get', 'remote.origin.url') requires_clone = repo_origin_url.strip('\n') != repository if requires_clone: rm('-r', directory)
def get_branches(repo_folder: Path, extra_args: tp.Optional[tp.List[str]] = None) -> str: """Show git branches.""" args = ["branch"] if extra_args: args += extra_args return tp.cast(str, git("-C", repo_folder.absolute(), args))
def versions_impl(): """Return a list of versions from the git hashes up to :limit:.""" directory = cls.SRC_FILE if target_dir is None else target_dir repo_prefix = local.path(str(CFG["tmp_dir"])) repo_loc = local.path(repo_prefix) / directory if source_required(repo_loc): if not clone: return [] git("clone", repo, repo_loc) update_hash(repo_loc) with local.cwd(repo_loc): rev_list = git("rev-list", "--abbrev-commit", refspec, *rev_list_args).strip().split('\n') latest = git("rev-parse", "--short=8", refspec).strip().split('\n') cls.VERSION = latest[0] return rev_list[:limit] if limit else rev_list
def push_current_branch(repo_folder: tp.Optional[Path] = None, upstream: tp.Optional[str] = None, branch_name: tp.Optional[str] = None) -> None: """Push in changes in a certain branch.""" cmd_args = ["push"] if upstream is not None: cmd_args.append("--set-upstream") cmd_args.append(upstream) if branch_name is not None: cmd_args.append(branch_name) else: cmd_args.append(get_current_branch(repo_folder)) if repo_folder is None or repo_folder == Path(""): git(cmd_args) else: git("-C", repo_folder.absolute(), cmd_args)
def get_head_commit(repo_folder: tp.Optional[Path] = None) -> FullCommitHash: """ Get the current HEAD commit. Args: repo_folder:where the git repository is located Returns: head commit hash """ return FullCommitHash( git(__get_git_path_arg(repo_folder), "rev-parse", "HEAD").strip())
def versions_impl(): """Return a list of versions from the git hashes up to :limit:.""" directory = cls.SRC_FILE if target_dir is None else target_dir repo_prefix = local.path(str(CFG["tmp_dir"])) repo_loc = local.path(repo_prefix) / directory if __clone_needed__(repo, repo_loc): if not clone: return [] git("clone", repo, repo_loc) with local.cwd(repo_loc): rev_list = git("rev-list", "--abbrev-commit", "--abbrev=10", refspec, *rev_list_args).strip().split('\n') latest = git("rev-parse", "--short=10", refspec).strip().split('\n') cls.VERSION = latest[0] if limit: return list(filter(version_filter, rev_list))[:limit] return list(filter(version_filter, rev_list))
def versions_impl(): """Return a list of versions from the git hashes up to :limit:.""" directory = cls.SRC_FILE if target_dir is None else target_dir repo_prefix = local.path(str(CFG["tmp_dir"])) repo_loc = local.path(repo_prefix) / directory if source_required(repo_loc): if not clone: return [] git("clone", repo, repo_loc) update_hash(repo_loc) with local.cwd(repo_loc): rev_list = git("rev-list", "--abbrev-commit", "--abbrev=10", refspec, *rev_list_args).strip().split('\n') latest = git("rev-parse", "--short=10", refspec).strip().split('\n') cls.VERSION = latest[0] if limit: return list(filter(version_filter, rev_list))[:limit] return list(filter(version_filter, rev_list))
def get_initial_commit( repo_folder: tp.Optional[Path] = None) -> FullCommitHash: """ Get the initial commit of a repository, i.e., the first commit made. Args: repo_folder: where the git repository is located Returns: initial commit hash """ return FullCommitHash( git(__get_git_path_arg(repo_folder), "rev-list", "--max-parents=0", "HEAD").strip())
def install_uchroot(): from benchbuild.utils.cmd import git, mkdir builddir = settings.CFG["build_dir"].value() with local.cwd(builddir): if not os.path.exists("erlent/.git"): git("clone", settings.CFG["uchroot"]["repo"].value()) else: with local.cwd("erlent"): git("pull", "--rebase") mkdir("-p", "erlent/build") with local.cwd("erlent/build"): from benchbuild.utils.cmd import cmake from benchbuild.utils.cmd import make cmake("../") make() erlent_path = os.path.abspath(os.path.join(builddir, "erlent", "build")) os.environ["PATH"] = os.path.pathsep.join( [erlent_path, os.environ["PATH"]]) local.env.update(PATH=os.environ["PATH"]) if not find_package("uchroot"): sys.exit(-1) settings.CFG["env"]["lookup_path"].value().append(erlent_path)
def get_current_branch(repo_folder: tp.Optional[Path] = None) -> str: """ Get the current branch of a repository, e.g., HEAD. Args: repo_folder: where the git repository is located Returns: branch name """ return tp.cast( str, git(__get_git_path_arg(repo_folder), "rev-parse", "--abbrev-ref", "HEAD").strip())
def get_tagged_commits(project_name: str) -> tp.List[tp.Tuple[str, str]]: """Get a list of all tagged commits along with their respective tags.""" repo_loc = get_local_project_git_path(project_name) with local.cwd(repo_loc): # --dereference resolves tag IDs into commits # These lines are indicated by the suffix '^{}' (see man git-show-ref) ref_list: tp.List[str] = git("show-ref", "--tags", "--dereference").strip().split("\n") ref_list = [ref for ref in ref_list if ref.endswith("^{}")] refs: tp.List[tp.Tuple[str, str]] = [ (ref_split[0], ref_split[1][10:-3]) for ref_split in [ref.strip().split() for ref in ref_list] ] return refs
def version(self, target_dir: str, version: str = 'HEAD') -> pb.LocalPath: """Overrides ``Git`` s version to create a new git worktree pointing to the requested version.""" main_repo_src_local = self.fetch() tgt_loc = pb.local.path(target_dir) / self.local vara_test_repos_path = self.__vara_test_repos_git.fetch() main_repo_src_remote = vara_test_repos_path / self.remote mkdir('-p', tgt_loc) # Extract main repository cp("-r", main_repo_src_local + "/.", tgt_loc) # Skip submodule extraction if none exist if not Path(tgt_loc / ".gitmodules").exists(): with pb.local.cwd(tgt_loc): git("checkout", "--detach", version) return tgt_loc # Extract submodules with pb.local.cwd(tgt_loc): # Get submodule entries submodule_url_entry_list = git( "config", "--file", ".gitmodules", "--name-only", "--get-regexp", "url" ).split('\n') # Remove empty strings submodule_url_entry_list = list( filter(None, submodule_url_entry_list) ) for entry in submodule_url_entry_list: relative_submodule_url = Path( git("config", "--file", ".gitmodules", "--get", entry).replace('\n', '') ) copy_renamed_git_to_dest( main_repo_src_remote / relative_submodule_url, relative_submodule_url ) git("checkout", "--detach", version) git("submodule", "update") return tgt_loc
def Git(repository, directory, rev=None, prefix=None, shallow_clone=True): """ Get a clone of the given repo Args: repository (str): Git URL of the SOURCE repo. directory (str): Name of the repo folder on disk. tgt_root (str): TARGET folder for the git repo. Defaults to ``CFG["tmpdir"]`` shallow_clone (bool): Only clone the repository shallow Defaults to true """ repository_loc = str(prefix) if prefix is None: repository_loc = str(CFG["tmp_dir"]) from benchbuild.utils.cmd import git src_dir = local.path(repository_loc) / directory if not source_required(src_dir): Copy(src_dir, ".") return extra_param = [] if shallow_clone: extra_param.append("--depth") extra_param.append("1") git("clone", extra_param, repository, src_dir) if rev: with local.cwd(src_dir): git("checkout", rev) update_hash(src_dir) Copy(src_dir, ".") return repository_loc
def Git(src_url, tgt_name, tgt_root=None): """ Get a shallow clone of the given repo Args: src_url (str): Git URL of the SOURCE repo. tgt_name (str): Name of the repo folder on disk. tgt_root (str): TARGET folder for the git repo. Defaults to ``CFG["tmpdir"]`` """ if tgt_root is None: tgt_root = str(CFG["tmp_dir"]) from os import path from benchbuild.utils.cmd import git src_dir = path.join(tgt_root, tgt_name) if not source_required(tgt_name, tgt_root): Copy(src_dir, ".") return git("clone", "--depth", "1", src_url, src_dir) update_hash(tgt_name, tgt_root) Copy(src_dir, ".")
def calc_code_churn( repo_path: Path, commit_a: FullCommitHash, commit_b: FullCommitHash, churn_config: tp.Optional[ChurnConfig] = None ) -> tp.Tuple[int, int, int]: """ Calculates churn between two commits. Args: repo: git repository commit_a: base commit for diff calculation commit_b: target commit for diff calculation churn_config: churn config to customize churn generation Returns: dict of churn triples, where the commit hash points to (files changed, insertions, deletions) """ churn_config = ChurnConfig.init_as_default_if_none(churn_config) diff_base_params = [ "diff", "--shortstat", "-l0", commit_a.hash, commit_b.hash ] if not churn_config.include_everything: diff_base_params.append("--") # builds a regex to select files that git includes into churn calc diff_base_params = diff_base_params + \ churn_config.get_extensions_repr('*.') stdout = git(__get_git_path_arg(repo_path), diff_base_params) # initialize with 0 as otherwise commits without changes would be # missing from the churn data match = GIT_DIFF_MATCHER.match(stdout) if match: def value_or_zero(match_result: tp.Any) -> int: if match_result is not None: return int(match_result) return 0 files_changed = value_or_zero(match.group('files')) insertions = value_or_zero(match.group('insertions')) deletions = value_or_zero(match.group('deletions')) return files_changed, insertions, deletions return 0, 0, 0
def get_tags(repo_folder: Path, extra_args: tp.Optional[tp.List[str]] = None) -> tp.List[str]: """Get the list of available git tags.""" args = ["tag"] if extra_args: args += extra_args git_tag_string: str = git("-C", repo_folder.absolute(), args) git_tag_list: tp.List[str] = [] if git_tag_string: git_tag_list = git_tag_string.split("\n") git_tag_list.remove('') return git_tag_list return git_tag_list
def get_git_hash(from_url): """ Get the git commit hash of HEAD from :from_url. Args: from_url: The file system url of our git repository. Returns: git commit hash of HEAD, or empty string. """ from benchbuild.utils.cmd import git if from_url is None: return "" if not path.exists(from_url): return "" with local.cwd(from_url): return git("rev-parse", "HEAD", retcode=None)
def download_repo(dl_folder: Path, url: str, repo_name: tp.Optional[str] = None, remote_name: tp.Optional[str] = None, post_out: tp.Callable[[str], None] = lambda x: None) -> None: """Download a repo into the specified folder.""" if not dl_folder.exists(): raise Exception(f"Could not find download folder {dl_folder}") args = ["clone", "--progress", url] if remote_name is not None: args.append("--origin") args.append(remote_name) if repo_name is not None: args.append(repo_name) output = git("-C", dl_folder, args) for line in output.split("\n"): post_out(line)
def get_commits_after_timestamp( timestamp: str, repo_folder: tp.Optional[Path] = None) -> tp.List[FullCommitHash]: """ Get all commits after a specific timestamp (given as a git date format). Note: for imprecise timestamps (e.g., only 2020), the day and month will default to today. Args: repo_folder: where the git repository is located timestamp: after which commits should be collected Returns: list[newest_commit, ..., last_commit_after_timestamp] """ return [ FullCommitHash(hash_val) for hash_val in git(__get_git_path_arg(repo_folder), "rev-list", f"--after={timestamp}", "HEAD").split() ]
def get_all_revisions_between( c_start: str, c_end: str, hash_type: tp.Type[CommitHashTy], repo_folder: tp.Optional[Path] = None) -> tp.List[CommitHashTy]: """ Returns a list of all revisions between two commits c_start and c_end (both inclusive), where c_start comes before c_end. It is assumed that the current working directory is the git repository. Args: c_start: first commit of the range c_end: last commit of the range short: shorten revision hashes repo_folder: where the git repository is located """ result = [c_start] result.extend( reversed( git(__get_git_path_arg(repo_folder), "log", "--pretty=%H", "--ancestry-path", f"{c_start}..{c_end}").strip().split())) return list(map(hash_type, result))
def generate_commit_map(path: Path, end: str = "HEAD", start: tp.Optional[str] = None, refspec: str = "HEAD") -> CommitMap: """ Generate a commit map for a repository including the commits. Range of commits that get included in the map: `]start..end]` Args: path: to the repository end: last commit that should be included start: parent of the first commit that should be included refspec: that should be checked out Returns: initalized ``CommitMap`` """ search_range = "" if start is not None: search_range += start + ".." search_range += end with local.cwd(path): old_head = get_current_branch() git("checkout", refspec) full_out = git("--no-pager", "log", "--pretty=format:'%H'") wanted_out = git("--no-pager", "log", "--pretty=format:'%H'", search_range) def format_stream() -> tp.Generator[str, None, None]: wanted_cm = set() for line in wanted_out.split('\n'): wanted_cm.add(line[1:-1]) for number, line in enumerate(reversed(full_out.split('\n'))): line = line[1:-1] if line in wanted_cm: yield f"{number}, {line}\n" git("checkout", old_head) return CommitMap(format_stream())