def remove_ref(self, name: str, old_ref: Optional[str] = None):
ref = self.repo.references.get(name)
if not ref:
raise SCMError(f"Ref '{name}' does not exist")
if old_ref and old_ref != str(ref.target):
raise SCMError(f"Failed to remove '{name}'")
ref.delete()
def merge(
self,
rev: str,
commit: bool = True,
msg: Optional[str] = None,
squash: bool = False,
) -> Optional[str]:
from git.exc import GitCommandError
if commit and squash:
raise SCMError("Cannot merge with 'squash' and 'commit'")
if commit and not msg:
raise SCMError("Merge commit message is required")
merge = partial(self.git.merge, rev)
try:
if commit:
merge(m=msg)
return self.get_rev()
merge(no_commit=True, squash=True)
except GitCommandError as exc:
if "CONFLICT" in str(exc):
raise MergeConflictError("Merge contained conflicts") from exc
raise SCMError("Merge failed") from exc
return None
def set_ref(
self,
name: str,
new_ref: str,
old_ref: Optional[str] = None,
message: Optional[str] = None,
symbolic: Optional[bool] = False,
):
from git.exc import GitCommandError
if old_ref and self.get_ref(name) != old_ref:
raise SCMError(f"Failed to set ref '{name}'")
try:
if symbolic:
if message:
self.git.symbolic_ref(name, new_ref, m=message)
else:
self.git.symbolic_ref(name, new_ref)
else:
args = [name, new_ref]
if old_ref:
args.append(old_ref)
if message:
self.git.update_ref(*args, m=message, create_reflog=True)
else:
self.git.update_ref(*args)
except GitCommandError as exc:
raise SCMError(f"Failed to set ref '{name}'") from exc
def remove_ref(self, name: str, old_ref: Optional[str] = None):
from git.exc import GitCommandError
if old_ref and self.get_ref(name) != old_ref:
raise SCMError(f"Failed to set ref '{name}'")
try:
args = [name]
if old_ref:
args.append(old_ref)
self.git.update_ref(*args, d=True)
except GitCommandError as exc:
raise SCMError(f"Failed to set ref '{name}'") from exc
def commit(self, msg: str, no_verify: bool = False):
from dulwich.errors import CommitError
from dulwich.porcelain import commit
from dulwich.repo import InvalidUserIdentity
try:
commit(self.root_dir, message=msg, no_verify=no_verify)
except CommitError as exc:
raise SCMError("Git commit failed") from exc
except InvalidUserIdentity as exc:
raise SCMError(
"Git username and email must be configured") from exc
def branch(self, branch: str):
from dulwich.porcelain import Error, branch_create
try:
branch_create(self.root_dir, branch)
except Error as exc:
raise SCMError(f"Failed to create branch '{branch}'") from exc
def commit(self, msg: str, no_verify: bool = False):
from git.exc import HookExecutionError
try:
self.repo.index.commit(msg, skip_hooks=no_verify)
except HookExecutionError as exc:
raise SCMError("Git pre-commit hook failed") from exc
def get_refs_containing(self, rev: str, pattern: Optional[str] = None):
import fnmatch
from pygit2 import GitError
def _contains(repo, ref, search_commit):
commit, _ref = self._resolve_refish(ref)
base = repo.merge_base(search_commit.id, commit.id)
return base == search_commit.id
try:
search_commit, _ref = self._resolve_refish(rev)
except (KeyError, GitError):
raise SCMError(f"Invalid rev '{rev}'")
if not pattern:
yield from (ref for ref in self.iter_refs()
if _contains(self.repo, ref, search_commit))
return
literal = pattern.rstrip("/").split("/")
for ref in self.iter_refs():
if (ref.split("/")[:len(literal)] == literal
or fnmatch.fnmatch(ref, pattern)) and _contains(
self.repo, ref, search_commit):
yield ref
def branch(self, branch: str):
from pygit2 import GitError
try:
commit = self.repo[self.repo.head.target]
self.repo.create_branch(branch, commit)
except GitError as exc:
raise SCMError(f"Failed to create branch '{branch}'") from exc
def pop(self):
logger.debug("Popping from stash '%s'", self.ref)
ref = f"{self.ref}@{{0}}"
rev = self.scm.resolve_rev(ref)
try:
self.apply(rev)
except Exception as exc:
raise SCMError("Could not apply stash commit") from exc
self.drop()
return rev
def _stash_drop(self, ref: str, index: int):
from dvc.scm.git import Stash
if ref == Stash.DEFAULT_STASH:
raise NotImplementedError
stash = self._get_stash(ref)
try:
stash.drop(index)
except ValueError as exc:
raise SCMError("Failed to drop stash entry") from exc
def _stash_apply(self, rev: str):
from git.exc import GitCommandError
try:
self.git.stash("apply", rev)
except GitCommandError as exc:
out = str(exc)
if "CONFLICT" in out or "already exists" in out:
raise MergeConflictError(
"Stash apply resulted in merge conflicts"
) from exc
raise SCMError("Could not apply stash") from exc
def merge(
self,
rev: str,
commit: bool = True,
msg: Optional[str] = None,
squash: bool = False,
) -> Optional[str]:
from pygit2 import GIT_RESET_MIXED, GitError
if commit and squash:
raise SCMError("Cannot merge with 'squash' and 'commit'")
if commit and not msg:
raise SCMError("Merge commit message is required")
with self.release_odb_handles():
try:
self.repo.index.read(False)
self.repo.merge(rev)
self.repo.index.write()
except GitError as exc:
raise SCMError("Merge failed") from exc
if self.repo.index.conflicts:
raise MergeConflictError("Merge contained conflicts")
if commit:
user = self.default_signature
tree = self.repo.index.write_tree()
merge_commit = self.repo.create_commit(
"HEAD", user, user, msg, tree,
[self.repo.head.target, rev])
return str(merge_commit)
if squash:
self.repo.reset(self.repo.head.target, GIT_RESET_MIXED)
self.repo.state_cleanup()
self.repo.index.write()
return None
def resolve_commit(self, rev: str) -> "GitCommit":
from pygit2 import GitError
try:
commit, _ref = self._resolve_refish(rev)
except (KeyError, GitError):
raise SCMError(f"Invalid commit '{rev}'")
return GitCommit(
str(commit.id),
commit.commit_time,
commit.commit_time_offset,
commit.message,
[str(parent) for parent in commit.parent_ids],
)
def __init__( # pylint:disable=W0231
self,
root_dir=os.curdir,
search_parent_directories=True):
from dulwich.errors import NotGitRepository
from dulwich.repo import Repo
try:
if search_parent_directories:
self.repo = Repo.discover(start=root_dir)
else:
self.repo = Repo(root_dir)
except NotGitRepository as exc:
raise SCMError(f"{root_dir} is not a git repository") from exc
self._submodules: Dict[str, str] = self._find_submodules()
self._stashes: dict = {}
def resolve_commit(self, rev: str) -> "GitCommit":
"""Return Commit object for the specified revision."""
from git.exc import BadName, GitCommandError
from git.objects.tag import TagObject
try:
commit = self.repo.rev_parse(rev)
except (BadName, GitCommandError):
raise SCMError(f"Invalid commit '{rev}'")
if isinstance(commit, TagObject):
commit = commit.object
return GitCommit(
commit.hexsha,
commit.committed_date,
commit.committer_tz_offset,
commit.message,
[str(parent) for parent in commit.parents],
)
def __init__( # pylint:disable=W0231
self, root_dir=os.curdir, search_parent_directories=True
):
import git
from git.exc import InvalidGitRepositoryError
try:
self.repo = git.Repo(
root_dir, search_parent_directories=search_parent_directories
)
except InvalidGitRepositoryError:
msg = "{} is not a git repository"
raise SCMError(msg.format(root_dir))
# NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
env = fix_env()
libpath = env.get("LD_LIBRARY_PATH", None)
self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
def set_ref(
self,
name: str,
new_ref: str,
old_ref: Optional[str] = None,
message: Optional[str] = None,
symbolic: Optional[bool] = False,
):
name_b = os.fsencode(name)
new_ref_b = os.fsencode(new_ref)
old_ref_b = os.fsencode(old_ref) if old_ref else None
message_b = message.encode("utf-8") if message else None
if symbolic:
return self.repo.refs.set_symbolic_ref(name_b,
new_ref_b,
message=message_b)
if not self.repo.refs.set_if_equals(
name_b, old_ref_b, new_ref_b, message=message_b):
raise SCMError(f"Failed to set '{name}'")
def __init__( # pylint:disable=W0231
self,
root_dir=os.curdir,
search_parent_directories=True):
import pygit2
if search_parent_directories:
ceiling_dirs = ""
else:
ceiling_dirs = os.path.abspath(root_dir)
# NOTE: discover_repository will return path/.git/
path = pygit2.discover_repository( # pylint:disable=no-member
root_dir, True, ceiling_dirs)
if not path:
raise SCMError(f"{root_dir} is not a git repository")
self.repo = pygit2.Repository(path)
self._stashes: dict = {}
def set_ref(
self,
name: str,
new_ref: str,
old_ref: Optional[str] = None,
message: Optional[str] = None,
symbolic: Optional[bool] = False,
):
if old_ref and old_ref != self.get_ref(name, follow=False):
raise SCMError(f"Failed to set '{name}'")
if message:
self._refdb.ensure_log(name)
if symbolic:
self.repo.create_reference_symbolic(name,
new_ref,
True,
message=message)
else:
self.repo.create_reference_direct(name,
new_ref,
True,
message=message)
def _stash_push(
self,
ref: str,
message: Optional[str] = None,
include_untracked: Optional[bool] = False,
) -> Tuple[Optional[str], bool]:
from dulwich.repo import InvalidUserIdentity
from dvc.scm.git import Stash
if include_untracked or ref == Stash.DEFAULT_STASH:
# dulwich stash.push does not support include_untracked and does
# not touch working tree
raise NotImplementedError
stash = self._get_stash(ref)
message_b = message.encode("utf-8") if message else None
try:
rev = stash.push(message=message_b)
except InvalidUserIdentity as exc:
raise SCMError(
"Git username and email must be configured") from exc
return os.fsdecode(rev), True
def get_rev(self) -> str:
rev = self.get_ref("HEAD")
if rev:
return rev
raise SCMError("Empty git repo")
def remove_ref(self, name: str, old_ref: Optional[str] = None):
name_b = name.encode("utf-8")
old_ref_b = old_ref.encode("utf-8") if old_ref else None
if not self.repo.refs.remove_if_equals(name_b, old_ref_b):
raise SCMError(f"Failed to remove '{name}'")
def push_refspec(
self,
url: str,
src: Optional[str],
dest: str,
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
progress: Callable[["GitProgressEvent"], None] = None,
**kwargs,
):
from dulwich.client import HTTPUnauthorized, get_transport_and_path
from dulwich.errors import NotGitRepository, SendPackError
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
dest_refs, values = self._push_dest_refs(src, dest)
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
def update_refs(refs):
from dulwich.objects import ZERO_SHA
new_refs = {}
for ref, value in zip(dest_refs, values):
if ref in refs and value != ZERO_SHA:
local_sha = self.repo.refs[ref]
remote_sha = refs[ref]
try:
check_diverged(self.repo, remote_sha, local_sha)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(ref), os.fsdecode(remote_sha))
if not overwrite:
continue
new_refs[ref] = value
return new_refs
try:
from dvc.scm.progress import GitProgressReporter
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=GitProgressReporter(progress) if progress else None,
)
except (NotGitRepository, SendPackError) as exc:
raise SCMError("Git failed to push '{src}' to '{url}'") from exc
except HTTPUnauthorized:
raise AuthError(url)
def fetch_refspecs(
self,
url: str,
refspecs: Iterable[str],
force: Optional[bool] = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
progress: Callable[["GitProgressEvent"], None] = None,
**kwargs,
):
from dulwich.client import get_transport_and_path
from dulwich.objectspec import parse_reftuples
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
fetch_refs = []
def determine_wants(remote_refs):
fetch_refs.extend(
parse_reftuples(
remote_refs,
self.repo.refs,
[os.fsencode(refspec) for refspec in refspecs],
force=force,
))
return [
remote_refs[lh] for (lh, _, _) in fetch_refs
if remote_refs[lh] not in self.repo.object_store
]
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
from dvc.scm.progress import GitProgressReporter
fetch_result = client.fetch(
path,
self.repo,
progress=GitProgressReporter(progress) if progress else None,
determine_wants=determine_wants,
)
for (lh, rh, _) in fetch_refs:
try:
if rh in self.repo.refs:
check_diverged(self.repo, self.repo.refs[rh],
fetch_result.refs[lh])
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(rh),
os.fsdecode(fetch_result.refs[lh]))
if not overwrite:
continue
self.repo.refs[rh] = fetch_result.refs[lh]
def default_signature(self):
try:
return self.repo.default_signature
except KeyError as exc:
raise SCMError(
"Git username and email must be configured") from exc
def drop(self, index: int = 0):
if index < 0 or index >= len(self):
raise SCMError(f"Invalid stash ref '{self.ref}@{{{index}}}'")
logger.debug("Dropping '%s@{%d}'", self.ref, index)
self.scm._stash_drop( # pylint: disable=protected-access
self.ref, index)
def pull(self, **kwargs):
infos = self.repo.remote().pull(**kwargs)
for info in infos:
if info.flags & info.ERROR:
raise SCMError(f"pull failed: {info.note}")
def push(self):
infos = self.repo.remote().push()
for info in infos:
if info.flags & info.ERROR:
raise SCMError(f"push failed: {info.summary}")