def fetch_refspecs(
self,
url: str,
refspecs: Iterable[str],
force: Optional[bool] = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
):
from dulwich.client import get_transport_and_path
from dulwich.objectspec import parse_reftuples
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
fetch_refs = []
def determine_wants(remote_refs):
fetch_refs.extend(
parse_reftuples(
remote_refs,
self.repo.refs,
[os.fsencode(refspec) for refspec in refspecs],
force=force,
))
return [
remote_refs[lh] for (lh, _, _) in fetch_refs
if remote_refs[lh] not in self.repo.object_store
]
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
def progress(msg):
logger.trace("git fetch: %s", msg)
fetch_result = client.fetch(path,
self.repo,
progress=progress,
determine_wants=determine_wants)
for (lh, rh, _) in fetch_refs:
try:
if rh in self.repo.refs:
check_diverged(self.repo, self.repo.refs[rh],
fetch_result.refs[lh])
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(rh),
os.fsdecode(fetch_result.refs[lh]))
if not overwrite:
continue
self.repo.refs[rh] = fetch_result.refs[lh]
def push_refspec(
self,
url: str,
src: Optional[str],
dest: str,
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
):
from dulwich.client import get_transport_and_path
from dulwich.errors import NotGitRepository, SendPackError
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
dest_refs, values = self._push_dest_refs(src, dest)
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL"
) from exc
def update_refs(refs):
new_refs = {}
for ref, value in zip(dest_refs, values):
if ref in refs:
local_sha = self.repo.refs[ref]
remote_sha = refs[ref]
try:
check_diverged(self.repo, remote_sha, local_sha)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(ref), os.fsdecode(remote_sha),
)
if not overwrite:
continue
new_refs[ref] = value
return new_refs
def progress(msg):
logger.trace("git send_pack: %s", msg)
try:
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=progress,
)
except (NotGitRepository, SendPackError) as exc:
raise SCMError("Git failed to push '{src}' to '{url}'") from exc
def validate_git_remote(self, url: str, **kwargs):
from dulwich.client import LocalGitClient, get_transport_and_path
from dulwich.porcelain import get_remote_repo
try:
_, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise InvalidRemoteSCMRepo(url) from exc
if isinstance(client, LocalGitClient) and not os.path.exists(
os.path.join("", path)):
raise InvalidRemoteSCMRepo(url)
def iter_remote_refs(self, url: str, base: Optional[str] = None):
from dulwich.client import get_transport_and_path
from dulwich.porcelain import get_remote_repo
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
if base:
yield from (os.fsdecode(ref) for ref in client.get_refs(path)
if ref.startswith(os.fsencode(base)))
else:
yield from (os.fsdecode(ref) for ref in client.get_refs(path))
def iter_remote_refs(self, url: str, base: Optional[str] = None, **kwargs):
from dulwich.client import HTTPUnauthorized, get_transport_and_path
from dulwich.errors import NotGitRepository
from dulwich.porcelain import get_remote_repo
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise InvalidRemoteSCMRepo(url) from exc
try:
if base:
yield from (os.fsdecode(ref) for ref in client.get_refs(path)
if ref.startswith(os.fsencode(base)))
else:
yield from (os.fsdecode(ref) for ref in client.get_refs(path))
except NotGitRepository as exc:
raise InvalidRemoteSCMRepo(url) from exc
except HTTPUnauthorized:
raise GitAuthError(url)
def fetch_refspecs(
self,
url: str,
refspecs: Iterable[str],
force: Optional[bool] = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
**kwargs,
):
from dulwich.client import get_transport_and_path
from dulwich.objectspec import parse_reftuples
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
fetch_refs = []
def determine_wants(remote_refs):
fetch_refs.extend(
parse_reftuples(
remote_refs,
self.repo.refs,
[os.fsencode(refspec) for refspec in refspecs],
force=force,
))
return [
remote_refs[lh] for (lh, _, _) in fetch_refs
if remote_refs[lh] not in self.repo.object_store
]
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
with Tqdm(desc="Fetching git refs",
bar_format=self.BAR_FMT_NOTOTAL) as pbar:
def progress(msg_b):
msg = msg_b.decode("ascii").strip()
pbar.update_msg(msg)
pbar.refresh()
logger.trace(msg)
fetch_result = client.fetch(
path,
self.repo,
progress=progress,
determine_wants=determine_wants,
)
for (lh, rh, _) in fetch_refs:
try:
if rh in self.repo.refs:
check_diverged(self.repo, self.repo.refs[rh],
fetch_result.refs[lh])
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(rh),
os.fsdecode(fetch_result.refs[lh]))
if not overwrite:
continue
self.repo.refs[rh] = fetch_result.refs[lh]
def push_refspec(
self,
url: str,
src: Optional[str],
dest: str,
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
**kwargs,
):
from dulwich.client import HTTPUnauthorized, get_transport_and_path
from dulwich.errors import NotGitRepository, SendPackError
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
dest_refs, values = self._push_dest_refs(src, dest)
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
def update_refs(refs):
from dulwich.objects import ZERO_SHA
new_refs = {}
for ref, value in zip(dest_refs, values):
if ref in refs and value != ZERO_SHA:
local_sha = self.repo.refs[ref]
remote_sha = refs[ref]
try:
check_diverged(self.repo, remote_sha, local_sha)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(ref), os.fsdecode(remote_sha))
if not overwrite:
continue
new_refs[ref] = value
return new_refs
try:
with Tqdm(desc="Pushing git refs",
bar_format=self.BAR_FMT_NOTOTAL) as pbar:
def progress(msg_b):
msg = msg_b.decode("ascii").strip()
pbar.update_msg(msg)
pbar.refresh()
logger.trace(msg)
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=progress,
)
except (NotGitRepository, SendPackError) as exc:
raise SCMError("Git failed to push '{src}' to '{url}'") from exc
except HTTPUnauthorized:
raise GitAuthError(url)
def push_refspec(
self,
url: str,
src: Optional[str],
dest: str,
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
progress: Callable[["GitProgressEvent"], None] = None,
**kwargs,
):
from dulwich.client import HTTPUnauthorized, get_transport_and_path
from dulwich.errors import NotGitRepository, SendPackError
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
dest_refs, values = self._push_dest_refs(src, dest)
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
def update_refs(refs):
from dulwich.objects import ZERO_SHA
new_refs = {}
for ref, value in zip(dest_refs, values):
if ref in refs and value != ZERO_SHA:
local_sha = self.repo.refs[ref]
remote_sha = refs[ref]
try:
check_diverged(self.repo, remote_sha, local_sha)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(ref), os.fsdecode(remote_sha))
if not overwrite:
continue
new_refs[ref] = value
return new_refs
try:
from dvc.scm.progress import GitProgressReporter
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=GitProgressReporter(progress) if progress else None,
)
except (NotGitRepository, SendPackError) as exc:
raise SCMError("Git failed to push '{src}' to '{url}'") from exc
except HTTPUnauthorized:
raise AuthError(url)
