def merge(
self,
rev: str,
commit: bool = True,
msg: Optional[str] = None,
squash: bool = False,
) -> Optional[str]:
from pygit2 import GIT_RESET_MIXED, GitError
if commit and squash:
raise SCMError("Cannot merge with 'squash' and 'commit'")
if commit and not msg:
raise SCMError("Merge commit message is required")
try:
self.repo.merge(rev)
except GitError as exc:
raise SCMError("Merge failed") from exc
if self.repo.index.conflicts:
raise MergeConflictError("Merge contained conflicts")
if commit:
user = self.repo.default_signature
tree = self.repo.index.write_tree()
merge_commit = self.repo.create_commit(
"HEAD", user, user, msg, tree, [self.repo.head.target, rev])
return str(merge_commit)
if squash:
self.repo.reset(self.repo.head.target, GIT_RESET_MIXED)
self.repo.state_cleanup()
return None
def merge(
self,
rev: str,
commit: bool = True,
msg: Optional[str] = None,
squash: bool = False,
) -> Optional[str]:
from git.exc import GitCommandError
if commit and squash:
raise SCMError("Cannot merge with 'squash' and 'commit'")
if commit and not msg:
raise SCMError("Merge commit message is required")
merge = partial(self.git.merge, rev)
try:
if commit:
merge(m=msg)
return self.get_rev()
merge(no_commit=True, squash=True)
except GitCommandError as exc:
if "CONFLICT" in str(exc):
raise MergeConflictError("Merge contained conflicts") from exc
raise SCMError("Merge failed") from exc
return None
def set_ref(
self,
name: str,
new_ref: str,
old_ref: Optional[str] = None,
message: Optional[str] = None,
symbolic: Optional[bool] = False,
):
from git.exc import GitCommandError
if old_ref and self.get_ref(name) != old_ref:
raise SCMError(f"Failed to set ref '{name}'")
try:
if symbolic:
if message:
self.git.symbolic_ref(name, new_ref, m=message)
else:
self.git.symbolic_ref(name, new_ref)
else:
args = [name, new_ref]
if old_ref:
args.append(old_ref)
if message:
self.git.update_ref(*args, m=message, create_reflog=True)
else:
self.git.update_ref(*args)
except GitCommandError as exc:
raise SCMError(f"Failed to set ref '{name}'") from exc
def remove_ref(self, name: str, old_ref: Optional[str] = None):
ref = self.repo.references.get(name)
if not ref:
raise SCMError(f"Ref '{name}' does not exist")
if old_ref and old_ref != str(ref.target):
raise SCMError(f"Failed to remove '{name}'")
ref.delete()
def push_refspec(
self,
url: str,
src: Optional[str],
dest: str,
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
):
from dulwich.client import get_transport_and_path
from dulwich.errors import NotGitRepository, SendPackError
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
dest_refs, values = self._push_dest_refs(src, dest)
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL"
) from exc
def update_refs(refs):
new_refs = {}
for ref, value in zip(dest_refs, values):
if ref in refs:
local_sha = self.repo.refs[ref]
remote_sha = refs[ref]
try:
check_diverged(self.repo, remote_sha, local_sha)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(ref), os.fsdecode(remote_sha),
)
if not overwrite:
continue
new_refs[ref] = value
return new_refs
def progress(msg):
logger.trace("git send_pack: %s", msg)
try:
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=progress,
)
except (NotGitRepository, SendPackError) as exc:
raise SCMError("Git failed to push '{src}' to '{url}'") from exc
def remove_ref(self, name: str, old_ref: Optional[str] = None):
from git.exc import GitCommandError
if old_ref and self.get_ref(name) != old_ref:
raise SCMError(f"Failed to set ref '{name}'")
try:
args = [name]
if old_ref:
args.append(old_ref)
self.git.update_ref(*args, d=True)
except GitCommandError as exc:
raise SCMError(f"Failed to set ref '{name}'") from exc
def commit(self, msg: str, no_verify: bool = False):
from dulwich.errors import CommitError
from dulwich.porcelain import commit
from dulwich.repo import InvalidUserIdentity
try:
commit(self.root_dir, message=msg, no_verify=no_verify)
except CommitError as exc:
raise SCMError("Git commit failed") from exc
except InvalidUserIdentity as exc:
raise SCMError(
"Git username and email must be configured") from exc
def push_refspec(self, url: str, src: Optional[str], dest: str):
from dulwich.client import get_transport_and_path
from dulwich.objects import ZERO_SHA
if src is not None and src.endswith("/"):
src_b = os.fsencode(src)
keys = self.repo.refs.subkeys(src_b)
values = [self.repo.refs[b"".join([src_b, key])] for key in keys]
dest_refs = [b"".join([os.fsencode(dest), key]) for key in keys]
else:
if src is None:
values = [ZERO_SHA]
else:
values = [self.repo.refs[os.fsencode(src)]]
dest_refs = [os.fsencode(dest)]
def update_refs(refs):
for ref, value in zip(dest_refs, values):
refs[ref] = value
return refs
try:
client, path = get_transport_and_path(url)
except Exception as exc:
raise SCMError("Could not get remote client") from exc
def progress(msg):
logger.trace("git send_pack: %s", msg)
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=progress,
)
def _get_diff_trees(self, a_ref, b_ref):
"""Private method for getting the trees and commit hashes of 2 git
references. Requires `gitdb` module (from gitpython package).
Args:
a_ref (str): git reference
b_ref (str): second git reference. If None, uses HEAD
Returns:
tuple: tuple with elements: (trees, commits)
"""
from gitdb.exc import BadObject, BadName
trees = {DIFF_A_TREE: None, DIFF_B_TREE: None}
commits = []
if b_ref is None:
b_ref = self.repo.head.commit
try:
a_commit = self.repo.git.rev_parse(a_ref, short=True)
b_commit = self.repo.git.rev_parse(b_ref, short=True)
# See https://gitpython.readthedocs.io
# /en/2.1.11/reference.html#git.objects.base.Object.__str__
commits.append(a_commit)
commits.append(b_commit)
trees[DIFF_A_TREE] = self.get_tree(commits[0])
trees[DIFF_B_TREE] = self.get_tree(commits[1])
except (BadName, BadObject) as exc:
raise SCMError("git problem") from exc
return trees, commits
def get_refs_containing(self, rev: str, pattern: Optional[str] = None):
import fnmatch
from pygit2 import GitError
def _contains(repo, ref, search_commit):
commit, _ref = self.repo.resolve_refish(ref)
base = repo.merge_base(search_commit.id, commit.id)
return base == search_commit.id
try:
search_commit, _ref = self.repo.resolve_refish(rev)
except (KeyError, GitError):
raise SCMError(f"Invalid rev '{rev}'")
if not pattern:
yield from (ref for ref in self.iter_refs()
if _contains(self.repo, ref, search_commit))
return
literal = pattern.rstrip("/").split("/")
for ref in self.iter_refs():
if (ref.split("/")[:len(literal)] == literal
or fnmatch.fnmatch(ref, pattern)) and _contains(
self.repo, ref, search_commit):
yield ref
def branch(self, branch: str):
from dulwich.porcelain import Error, branch_create
try:
branch_create(self.root_dir, branch)
except Error as exc:
raise SCMError(f"Failed to create branch '{branch}'") from exc
def __init__(self, root_dir=os.curdir, search_parent_directories=True):
"""Git class constructor.
Requires `Repo` class from `git` module (from gitpython package).
"""
super().__init__(root_dir)
import git
from git.exc import InvalidGitRepositoryError
try:
self.repo = git.Repo(
root_dir, search_parent_directories=search_parent_directories
)
except InvalidGitRepositoryError:
msg = "{} is not a git repository"
raise SCMError(msg.format(root_dir))
# NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
env = fix_env(None)
libpath = env.get("LD_LIBRARY_PATH", None)
self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
self.ignored_paths = []
self.files_to_track = set()
def drop(self, index: int = 0):
if index < 0 or index >= len(self):
raise SCMError(f"Invalid stash ref '{self.ref}@{{{index}}}'")
logger.debug("Dropping '%s@{%d}'", self.ref, index)
self.scm._stash_drop( # pylint: disable=protected-access
self.ref, index
)
def commit(self, msg: str, no_verify: bool = False):
from git.exc import HookExecutionError
try:
self.repo.index.commit(msg, skip_hooks=no_verify)
except HookExecutionError as exc:
raise SCMError("Git pre-commit hook failed") from exc
def default_signature(self):
try:
return self.repo.default_signature
except KeyError as exc:
raise SCMError(
"Git username and email must be configured"
) from exc
def fetch_refspecs(
self,
url: str,
refspecs: Iterable[str],
force: Optional[bool] = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
):
from dulwich.client import get_transport_and_path
from dulwich.objectspec import parse_reftuples
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
fetch_refs = []
def determine_wants(remote_refs):
fetch_refs.extend(
parse_reftuples(
remote_refs,
self.repo.refs,
[os.fsencode(refspec) for refspec in refspecs],
force=force,
))
return [
remote_refs[lh] for (lh, _, _) in fetch_refs
if remote_refs[lh] not in self.repo.object_store
]
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
def progress(msg):
logger.trace("git fetch: %s", msg)
fetch_result = client.fetch(path,
self.repo,
progress=progress,
determine_wants=determine_wants)
for (lh, rh, _) in fetch_refs:
try:
if rh in self.repo.refs:
check_diverged(self.repo, self.repo.refs[rh],
fetch_result.refs[lh])
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(rh),
os.fsdecode(fetch_result.refs[lh]))
if not overwrite:
continue
self.repo.refs[rh] = fetch_result.refs[lh]
def branch(self, branch: str):
from pygit2 import GitError
try:
commit = self.repo[self.repo.head.target]
self.repo.create_branch(branch, commit)
except GitError as exc:
raise SCMError(f"Failed to create branch '{branch}'") from exc
def _install_hook(self, name, cmd):
hook = os.path.join(self.root_dir, self.GIT_DIR, "hooks", name)
if os.path.isfile(hook):
msg = "git hook '{}' already exists."
raise SCMError(msg.format(os.path.relpath(hook)))
with open(hook, "w+") as fobj:
fobj.write("#!/bin/sh\nexec dvc {}\n".format(cmd))
os.chmod(hook, 0o777)
def commit(self, msg: str, no_verify: bool = False):
from dulwich.errors import CommitError
from dulwich.porcelain import commit
try:
commit(self.root_dir, message=msg, no_verify=no_verify)
except CommitError as exc:
raise SCMError("Git commit failed") from exc
def _stash_apply(self, rev: str):
from git.exc import GitCommandError
try:
self.git.stash("apply", rev)
except GitCommandError as exc:
if "CONFLICT" in str(exc):
raise MergeConflictError(
"Stash apply resulted in merge conflicts") from exc
raise SCMError("Could not apply stash") from exc
def pop(self):
logger.debug("Popping from stash '%s'", self.ref)
ref = "{0}@{{0}}".format(self.ref)
rev = self.scm.resolve_rev(ref)
try:
self.apply(rev)
except Exception as exc:
raise SCMError("Could not apply stash commit") from exc
self.drop()
return rev
def _stash_drop(self, ref: str, index: int):
from dvc.scm.git import Stash
if ref == Stash.DEFAULT_STASH:
raise NotImplementedError
stash = self._get_stash(ref)
try:
stash.drop(index)
except ValueError as exc:
raise SCMError("Failed to drop stash entry") from exc
def drop(self, index: int = 0):
ref = "{0}@{{{1}}}".format(self.ref, index)
if index < 0 or index >= len(self):
raise SCMError(f"Invalid stash ref '{ref}'")
logger.debug("Dropping '%s'", ref)
self.scm.reflog_delete(ref, updateref=True)
# if we removed the last reflog entry, delete the ref and reflog
if len(self) == 0:
self.scm.remove_ref(self.ref)
parts = self.ref.split("/")
reflog = os.path.join(self.scm.root_dir, ".git", "logs", *parts)
remove(reflog)
def resolve_commit(self, rev: str) -> "GitCommit":
from pygit2 import GitError
try:
commit, _ref = self.repo.resolve_refish(rev)
except (KeyError, GitError):
raise SCMError(f"Invalid commit '{rev}'")
return GitCommit(
str(commit.id),
commit.commit_time,
commit.commit_time_offset,
commit.message,
[str(parent) for parent in commit.parent_ids],
)
def __init__( # pylint:disable=W0231
self, root_dir=os.curdir, search_parent_directories=True
):
from dulwich.errors import NotGitRepository
from dulwich.repo import Repo
try:
if search_parent_directories:
self.repo = Repo.discover(start=root_dir)
else:
self.repo = Repo(root_dir)
except NotGitRepository as exc:
raise SCMError(f"{root_dir} is not a git repository") from exc
self._stashes: dict = {}
def iter_remote_refs(self, url: str, base: Optional[str] = None):
from dulwich.client import get_transport_and_path
from dulwich.porcelain import get_remote_repo
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
if base:
yield from (os.fsdecode(ref) for ref in client.get_refs(path)
if ref.startswith(os.fsencode(base)))
else:
yield from (os.fsdecode(ref) for ref in client.get_refs(path))
def set_ref(
self,
name: str,
new_ref: str,
old_ref: Optional[str] = None,
message: Optional[str] = None,
symbolic: Optional[bool] = False,
):
if old_ref and old_ref != self.get_ref(name, follow=False):
raise SCMError(f"Failed to set '{name}'")
if symbolic:
ref = self.repo.create_reference_symbolic(name, new_ref, True)
else:
ref = self.repo.create_reference_direct(name, new_ref, True)
if message:
ref.set_target(new_ref, message)
def __init__(self, root_dir=os.curdir, project=None):
super(Git, self).__init__(root_dir, project=project)
import git
from git.exc import InvalidGitRepositoryError
try:
self.repo = git.Repo(root_dir)
except InvalidGitRepositoryError:
msg = "{} is not a git repository"
raise SCMError(msg.format(root_dir))
# NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
env = fix_env(None)
libpath = env.get("LD_LIBRARY_PATH", None)
self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
def resolve_commit(self, rev: str) -> "GitCommit":
"""Return Commit object for the specified revision."""
from git.exc import BadName, GitCommandError
from git.objects.tag import TagObject
try:
commit = self.repo.rev_parse(rev)
except (BadName, GitCommandError):
raise SCMError(f"Invalid commit '{rev}'")
if isinstance(commit, TagObject):
commit = commit.object
return GitCommit(
commit.hexsha,
commit.committed_date,
commit.committer_tz_offset,
commit.message,
[str(parent) for parent in commit.parents],
)
def set_ref(
self,
name: str,
new_ref: str,
old_ref: Optional[str] = None,
message: Optional[str] = None,
symbolic: Optional[bool] = False,
):
name_b = os.fsencode(name)
new_ref_b = os.fsencode(new_ref)
old_ref_b = os.fsencode(old_ref) if old_ref else None
message_b = message.encode("utf-8") if message else None
if symbolic:
return self.repo.refs.set_symbolic_ref(name_b,
new_ref_b,
message=message_b)
if not self.repo.refs.set_if_equals(
name_b, old_ref_b, new_ref_b, message=message_b):
raise SCMError(f"Failed to set '{name}'")