def download_impl(self):
"""Download the selected version."""
nonlocal target_dir, git
directory = cls.SRC_FILE if target_dir is None else target_dir
Git(self.repository, directory)
with local.cwd(directory):
git("checkout", self.version)
def download_impl(self):
"""Download the selected version."""
nonlocal target_dir, git
directory = cls.SRC_FILE if target_dir is None else target_dir
Git(self.repository, directory)
with local.cwd(directory):
git("checkout", self.version)
def install_uchroot(_):
"""Installer for erlent (contains uchroot)."""
builddir = local.path(str(CFG["build_dir"].value))
with local.cwd(builddir):
erlent_src = local.path('erlent')
erlent_git = erlent_src / '.git'
erlent_repo = str(CFG['uchroot']['repo'])
erlent_build = erlent_src / 'build'
if not erlent_git.exists():
git("clone", erlent_repo)
else:
with local.cwd(erlent_src):
git("pull", "--rebase")
erlent_build.mkdir()
with local.cwd(erlent_build):
cmake("../")
make()
os.environ["PATH"] = os.path.pathsep.join(
[erlent_build, os.environ["PATH"]])
local.env.update(PATH=os.environ["PATH"])
if not find_package("uchroot"):
LOG.error('uchroot not found, after updating PATH to %s',
os.environ['PATH'])
sys.exit(-1)
env = CFG['env'].value
if 'PATH' not in env:
env['PATH'] = []
env['PATH'].append(str(erlent_build))
def test_vara_test_repo_lib_checkout(self) -> None:
"""Test if the repositories are checked out at the specified
revision."""
self.elementalist.version(
f"{self.bb_result_report_path}/"
f"TwoLibsOneProjectInteractionDiscreteLibsSingleProject"
f"-cpp_projects@{self.revision}",
version=self.revision)
# Are repositories checked out at correct commit hash?
with local.cwd(self.bb_result_lib_path / "Elementalist"):
self.assertEqual(self.revision[:7],
git('rev-parse', '--short', 'HEAD').rstrip())
with local.cwd(self.bb_result_lib_path / "fire_lib"):
self.assertEqual("ead5e00",
git('rev-parse', '--short', 'HEAD').rstrip())
with local.cwd(self.bb_result_lib_path / "water_lib"):
self.assertEqual("58ec513",
git('rev-parse', '--short', 'HEAD').rstrip())
with local.cwd(self.bb_result_lib_path / "earth_lib"):
self.assertEqual("1db6fbe",
git('rev-parse', '--short', 'HEAD').rstrip())
def calc_repo_loc(repo: pygit2.Repository, rev_range: str) -> int:
"""
Calculate the LOC for a project at its HEAD.
Args:
repo: the repository to calculate the LOC for
Returns:
the number of lines in source-code files
"""
project_path = repo.path[:-5]
churn_config = ChurnConfig.create_c_style_languages_config()
file_pattern = re.compile("|".join(
churn_config.get_extensions_repr(r"^.*\.", r"$")))
loc: int = 0
with local.cwd(project_path):
files = git(
"ls-tree",
"-r",
"--name-only",
rev_range,
).splitlines()
for file in files:
if file_pattern.match(file):
lines = git("show", f"{rev_range}:{file}").splitlines()
loc += len([line for line in lines if line])
return loc
def checkout_new_branch(repo_folder: Path,
branch: str,
remote_branch: tp.Optional[str] = None) -> None:
"""Checks out a new branch in the repository."""
args = ["checkout", "-b", branch]
if remote_branch is not None:
args.append(remote_branch)
git("-C", repo_folder.absolute(), args)
def copy_to_env(self, path: Path) -> None:
with self.__lock:
bb_tmp = str(path / "benchbuild/tmp")
settings.bb_cfg()["tmp_dir"] = bb_tmp
base.CFG["tmp_dir"] = bb_tmp
git(
"clone", "--dissociate", "--recurse-submodules", "--reference",
self.__local, self.__remote, f"{bb_tmp}/{self.__repo_name}"
)
def fetch_remote(remote: tp.Optional[str] = None,
repo_folder: tp.Optional[Path] = None,
extra_args: tp.Optional[tp.List[str]] = None) -> None:
"""Fetches the new changes from the remote."""
args = ["fetch"]
if extra_args:
args += extra_args
if remote:
args.append(remote)
git("-C", repo_folder, args)
def compile(self):
self.download()
with local.cwd(self.src_file):
git("fetch", "origin", "pull/17/head:clang")
git("checkout", "clang")
run.run(make["config"])
clang = compiler.cc(self)
clang_cxx = compiler.cxx(self)
run.run(make["CC=" + str(clang), "CXX=" + str(clang_cxx), "clean",
"lsh", "sh"])
def compile(self) -> None:
coreutils_source = local.path(self.source_of_primary)
compiler = bb.compiler.cc(self)
with local.cwd(coreutils_source):
git("submodule", "init")
git("submodule", "update")
with local.env(CC=str(compiler)):
bb.watch(local["./bootstrap"])()
bb.watch(local["./configure"])("--disable-gcc-warnings")
bb.watch(make)("-j", get_number_of_jobs(bb_cfg()))
verify_binaries(self)
def get_submodule_head(project_name: str, submodule_name: str,
commit: FullCommitHash) -> FullCommitHash:
"""
Retrieve the checked out commit for a submodule of a project.
Args:
project_name: name of the project
submodule_name: name of the submodule
commit: commit of the project's main repo
Returns:
checked out commit of the submodule
"""
if submodule_name == get_primary_project_source(project_name).local:
return commit
main_repo = get_local_project_git_path(project_name)
submodule_status = git(__get_git_path_arg(main_repo), "ls-tree", commit)
commit_pattern = re.compile(r"[0-9]* commit ([0-9abcdef]*)\t" +
submodule_name)
match = commit_pattern.search(submodule_status)
if match:
return FullCommitHash(match.group(1))
raise AssertionError(f"Unknown submodule {submodule_name}")
def __clone_needed__(repository: str, directory: str) -> bool:
"""
Do we need to create a fresh clone of the given repository.
Args:
repository: the repository we want to clone.
directory: the directory we expect the clone to live.
Returns:
True, if the clone is required.
False, if the directory is a valid clone.
"""
from benchbuild.utils.cmd import git, rm
git_dir = local.path(directory) / '.git'
if not git_dir.exists():
return True
requires_clone = True
with local.cwd(directory):
repo_origin_url = git('config', '--get', 'remote.origin.url')
requires_clone = repo_origin_url.strip('\n') != repository
if requires_clone:
rm('-r', directory)
def get_branches(repo_folder: Path,
extra_args: tp.Optional[tp.List[str]] = None) -> str:
"""Show git branches."""
args = ["branch"]
if extra_args:
args += extra_args
return tp.cast(str, git("-C", repo_folder.absolute(), args))
def versions_impl():
"""Return a list of versions from the git hashes up to :limit:."""
directory = cls.SRC_FILE if target_dir is None else target_dir
repo_prefix = local.path(str(CFG["tmp_dir"]))
repo_loc = local.path(repo_prefix) / directory
if source_required(repo_loc):
if not clone:
return []
git("clone", repo, repo_loc)
update_hash(repo_loc)
with local.cwd(repo_loc):
rev_list = git("rev-list", "--abbrev-commit", refspec,
*rev_list_args).strip().split('\n')
latest = git("rev-parse", "--short=8",
refspec).strip().split('\n')
cls.VERSION = latest[0]
return rev_list[:limit] if limit else rev_list
def push_current_branch(repo_folder: tp.Optional[Path] = None,
upstream: tp.Optional[str] = None,
branch_name: tp.Optional[str] = None) -> None:
"""Push in changes in a certain branch."""
cmd_args = ["push"]
if upstream is not None:
cmd_args.append("--set-upstream")
cmd_args.append(upstream)
if branch_name is not None:
cmd_args.append(branch_name)
else:
cmd_args.append(get_current_branch(repo_folder))
if repo_folder is None or repo_folder == Path(""):
git(cmd_args)
else:
git("-C", repo_folder.absolute(), cmd_args)
def get_head_commit(repo_folder: tp.Optional[Path] = None) -> FullCommitHash:
"""
Get the current HEAD commit.
Args:
repo_folder:where the git repository is located
Returns: head commit hash
"""
return FullCommitHash(
git(__get_git_path_arg(repo_folder), "rev-parse", "HEAD").strip())
def versions_impl():
"""Return a list of versions from the git hashes up to :limit:."""
directory = cls.SRC_FILE if target_dir is None else target_dir
repo_prefix = local.path(str(CFG["tmp_dir"]))
repo_loc = local.path(repo_prefix) / directory
if __clone_needed__(repo, repo_loc):
if not clone:
return []
git("clone", repo, repo_loc)
with local.cwd(repo_loc):
rev_list = git("rev-list", "--abbrev-commit", "--abbrev=10",
refspec, *rev_list_args).strip().split('\n')
latest = git("rev-parse", "--short=10",
refspec).strip().split('\n')
cls.VERSION = latest[0]
if limit:
return list(filter(version_filter, rev_list))[:limit]
return list(filter(version_filter, rev_list))
def versions_impl():
"""Return a list of versions from the git hashes up to :limit:."""
directory = cls.SRC_FILE if target_dir is None else target_dir
repo_prefix = local.path(str(CFG["tmp_dir"]))
repo_loc = local.path(repo_prefix) / directory
if source_required(repo_loc):
if not clone:
return []
git("clone", repo, repo_loc)
update_hash(repo_loc)
with local.cwd(repo_loc):
rev_list = git("rev-list", "--abbrev-commit", "--abbrev=10",
refspec, *rev_list_args).strip().split('\n')
latest = git("rev-parse", "--short=10",
refspec).strip().split('\n')
cls.VERSION = latest[0]
if limit:
return list(filter(version_filter, rev_list))[:limit]
return list(filter(version_filter, rev_list))
def get_initial_commit(
repo_folder: tp.Optional[Path] = None) -> FullCommitHash:
"""
Get the initial commit of a repository, i.e., the first commit made.
Args:
repo_folder: where the git repository is located
Returns: initial commit hash
"""
return FullCommitHash(
git(__get_git_path_arg(repo_folder), "rev-list", "--max-parents=0",
"HEAD").strip())
def install_uchroot():
from benchbuild.utils.cmd import git, mkdir
builddir = settings.CFG["build_dir"].value()
with local.cwd(builddir):
if not os.path.exists("erlent/.git"):
git("clone", settings.CFG["uchroot"]["repo"].value())
else:
with local.cwd("erlent"):
git("pull", "--rebase")
mkdir("-p", "erlent/build")
with local.cwd("erlent/build"):
from benchbuild.utils.cmd import cmake
from benchbuild.utils.cmd import make
cmake("../")
make()
erlent_path = os.path.abspath(os.path.join(builddir, "erlent", "build"))
os.environ["PATH"] = os.path.pathsep.join(
[erlent_path, os.environ["PATH"]])
local.env.update(PATH=os.environ["PATH"])
if not find_package("uchroot"):
sys.exit(-1)
settings.CFG["env"]["lookup_path"].value().append(erlent_path)
def get_current_branch(repo_folder: tp.Optional[Path] = None) -> str:
"""
Get the current branch of a repository, e.g., HEAD.
Args:
repo_folder: where the git repository is located
Returns: branch name
"""
return tp.cast(
str,
git(__get_git_path_arg(repo_folder), "rev-parse", "--abbrev-ref",
"HEAD").strip())
def get_tagged_commits(project_name: str) -> tp.List[tp.Tuple[str, str]]:
"""Get a list of all tagged commits along with their respective tags."""
repo_loc = get_local_project_git_path(project_name)
with local.cwd(repo_loc):
# --dereference resolves tag IDs into commits
# These lines are indicated by the suffix '^{}' (see man git-show-ref)
ref_list: tp.List[str] = git("show-ref", "--tags",
"--dereference").strip().split("\n")
ref_list = [ref for ref in ref_list if ref.endswith("^{}")]
refs: tp.List[tp.Tuple[str, str]] = [
(ref_split[0], ref_split[1][10:-3])
for ref_split in [ref.strip().split() for ref in ref_list]
]
return refs
def version(self, target_dir: str, version: str = 'HEAD') -> pb.LocalPath:
"""Overrides ``Git`` s version to create a new git worktree pointing to
the requested version."""
main_repo_src_local = self.fetch()
tgt_loc = pb.local.path(target_dir) / self.local
vara_test_repos_path = self.__vara_test_repos_git.fetch()
main_repo_src_remote = vara_test_repos_path / self.remote
mkdir('-p', tgt_loc)
# Extract main repository
cp("-r", main_repo_src_local + "/.", tgt_loc)
# Skip submodule extraction if none exist
if not Path(tgt_loc / ".gitmodules").exists():
with pb.local.cwd(tgt_loc):
git("checkout", "--detach", version)
return tgt_loc
# Extract submodules
with pb.local.cwd(tgt_loc):
# Get submodule entries
submodule_url_entry_list = git(
"config", "--file", ".gitmodules", "--name-only",
"--get-regexp", "url"
).split('\n')
# Remove empty strings
submodule_url_entry_list = list(
filter(None, submodule_url_entry_list)
)
for entry in submodule_url_entry_list:
relative_submodule_url = Path(
git("config", "--file", ".gitmodules", "--get",
entry).replace('\n', '')
)
copy_renamed_git_to_dest(
main_repo_src_remote / relative_submodule_url,
relative_submodule_url
)
git("checkout", "--detach", version)
git("submodule", "update")
return tgt_loc
def Git(repository, directory, rev=None, prefix=None, shallow_clone=True):
"""
Get a clone of the given repo
Args:
repository (str): Git URL of the SOURCE repo.
directory (str): Name of the repo folder on disk.
tgt_root (str): TARGET folder for the git repo.
Defaults to ``CFG["tmpdir"]``
shallow_clone (bool): Only clone the repository shallow
Defaults to true
"""
repository_loc = str(prefix)
if prefix is None:
repository_loc = str(CFG["tmp_dir"])
from benchbuild.utils.cmd import git
src_dir = local.path(repository_loc) / directory
if not source_required(src_dir):
Copy(src_dir, ".")
return
extra_param = []
if shallow_clone:
extra_param.append("--depth")
extra_param.append("1")
git("clone", extra_param, repository, src_dir)
if rev:
with local.cwd(src_dir):
git("checkout", rev)
update_hash(src_dir)
Copy(src_dir, ".")
return repository_loc
def Git(repository, directory, rev=None, prefix=None, shallow_clone=True):
"""
Get a clone of the given repo
Args:
repository (str): Git URL of the SOURCE repo.
directory (str): Name of the repo folder on disk.
tgt_root (str): TARGET folder for the git repo.
Defaults to ``CFG["tmpdir"]``
shallow_clone (bool): Only clone the repository shallow
Defaults to true
"""
repository_loc = str(prefix)
if prefix is None:
repository_loc = str(CFG["tmp_dir"])
from benchbuild.utils.cmd import git
src_dir = local.path(repository_loc) / directory
if not source_required(src_dir):
Copy(src_dir, ".")
return
extra_param = []
if shallow_clone:
extra_param.append("--depth")
extra_param.append("1")
git("clone", extra_param, repository, src_dir)
if rev:
with local.cwd(src_dir):
git("checkout", rev)
update_hash(src_dir)
Copy(src_dir, ".")
return repository_loc
def Git(src_url, tgt_name, tgt_root=None):
"""
Get a shallow clone of the given repo
Args:
src_url (str): Git URL of the SOURCE repo.
tgt_name (str): Name of the repo folder on disk.
tgt_root (str): TARGET folder for the git repo.
Defaults to ``CFG["tmpdir"]``
"""
if tgt_root is None:
tgt_root = str(CFG["tmp_dir"])
from os import path
from benchbuild.utils.cmd import git
src_dir = path.join(tgt_root, tgt_name)
if not source_required(tgt_name, tgt_root):
Copy(src_dir, ".")
return
git("clone", "--depth", "1", src_url, src_dir)
update_hash(tgt_name, tgt_root)
Copy(src_dir, ".")
def calc_code_churn(
repo_path: Path,
commit_a: FullCommitHash,
commit_b: FullCommitHash,
churn_config: tp.Optional[ChurnConfig] = None
) -> tp.Tuple[int, int, int]:
"""
Calculates churn between two commits.
Args:
repo: git repository
commit_a: base commit for diff calculation
commit_b: target commit for diff calculation
churn_config: churn config to customize churn generation
Returns:
dict of churn triples, where the commit hash points to
(files changed, insertions, deletions)
"""
churn_config = ChurnConfig.init_as_default_if_none(churn_config)
diff_base_params = [
"diff", "--shortstat", "-l0", commit_a.hash, commit_b.hash
]
if not churn_config.include_everything:
diff_base_params.append("--")
# builds a regex to select files that git includes into churn calc
diff_base_params = diff_base_params + \
churn_config.get_extensions_repr('*.')
stdout = git(__get_git_path_arg(repo_path), diff_base_params)
# initialize with 0 as otherwise commits without changes would be
# missing from the churn data
match = GIT_DIFF_MATCHER.match(stdout)
if match:
def value_or_zero(match_result: tp.Any) -> int:
if match_result is not None:
return int(match_result)
return 0
files_changed = value_or_zero(match.group('files'))
insertions = value_or_zero(match.group('insertions'))
deletions = value_or_zero(match.group('deletions'))
return files_changed, insertions, deletions
return 0, 0, 0
def get_tags(repo_folder: Path,
extra_args: tp.Optional[tp.List[str]] = None) -> tp.List[str]:
"""Get the list of available git tags."""
args = ["tag"]
if extra_args:
args += extra_args
git_tag_string: str = git("-C", repo_folder.absolute(), args)
git_tag_list: tp.List[str] = []
if git_tag_string:
git_tag_list = git_tag_string.split("\n")
git_tag_list.remove('')
return git_tag_list
return git_tag_list
def get_git_hash(from_url):
"""
Get the git commit hash of HEAD from :from_url.
Args:
from_url: The file system url of our git repository.
Returns:
git commit hash of HEAD, or empty string.
"""
from benchbuild.utils.cmd import git
if from_url is None:
return ""
if not path.exists(from_url):
return ""
with local.cwd(from_url):
return git("rev-parse", "HEAD", retcode=None)
def download_repo(dl_folder: Path,
url: str,
repo_name: tp.Optional[str] = None,
remote_name: tp.Optional[str] = None,
post_out: tp.Callable[[str], None] = lambda x: None) -> None:
"""Download a repo into the specified folder."""
if not dl_folder.exists():
raise Exception(f"Could not find download folder {dl_folder}")
args = ["clone", "--progress", url]
if remote_name is not None:
args.append("--origin")
args.append(remote_name)
if repo_name is not None:
args.append(repo_name)
output = git("-C", dl_folder, args)
for line in output.split("\n"):
post_out(line)
def get_commits_after_timestamp(
timestamp: str,
repo_folder: tp.Optional[Path] = None) -> tp.List[FullCommitHash]:
"""
Get all commits after a specific timestamp (given as a git date format).
Note: for imprecise timestamps (e.g., only 2020), the day and month will
default to today.
Args:
repo_folder: where the git repository is located
timestamp: after which commits should be collected
Returns: list[newest_commit, ..., last_commit_after_timestamp]
"""
return [
FullCommitHash(hash_val)
for hash_val in git(__get_git_path_arg(repo_folder), "rev-list",
f"--after={timestamp}", "HEAD").split()
]
def get_all_revisions_between(
c_start: str,
c_end: str,
hash_type: tp.Type[CommitHashTy],
repo_folder: tp.Optional[Path] = None) -> tp.List[CommitHashTy]:
"""
Returns a list of all revisions between two commits c_start and c_end (both
inclusive), where c_start comes before c_end.
It is assumed that the current working directory is the git repository.
Args:
c_start: first commit of the range c_end: last commit of the range
short: shorten revision hashes repo_folder: where the git repository is
located
"""
result = [c_start]
result.extend(
reversed(
git(__get_git_path_arg(repo_folder), "log", "--pretty=%H",
"--ancestry-path", f"{c_start}..{c_end}").strip().split()))
return list(map(hash_type, result))
def generate_commit_map(path: Path,
end: str = "HEAD",
start: tp.Optional[str] = None,
refspec: str = "HEAD") -> CommitMap:
"""
Generate a commit map for a repository including the commits.
Range of commits that get included in the map: `]start..end]`
Args:
path: to the repository
end: last commit that should be included
start: parent of the first commit that should be included
refspec: that should be checked out
Returns: initalized ``CommitMap``
"""
search_range = ""
if start is not None:
search_range += start + ".."
search_range += end
with local.cwd(path):
old_head = get_current_branch()
git("checkout", refspec)
full_out = git("--no-pager", "log", "--pretty=format:'%H'")
wanted_out = git("--no-pager", "log", "--pretty=format:'%H'",
search_range)
def format_stream() -> tp.Generator[str, None, None]:
wanted_cm = set()
for line in wanted_out.split('\n'):
wanted_cm.add(line[1:-1])
for number, line in enumerate(reversed(full_out.split('\n'))):
line = line[1:-1]
if line in wanted_cm:
yield f"{number}, {line}\n"
git("checkout", old_head)
return CommitMap(format_stream())
