def get_refs(self, remote):
self.export_commits()
client, path = self.get_transport_and_path(remote)
old_refs = {}
new_refs = {}
def changed(refs):
old_refs.update(refs)
to_push = set(self.local_heads().values() + self.tags.values())
new_refs.update(self.get_changed_refs(refs, to_push, True))
# don't push anything
return {}
try:
client.send_pack(path, changed, None)
changed_refs = [ref for ref, sha in new_refs.iteritems()
if sha != old_refs.get(ref)]
new = [bin(self.map_hg_get(new_refs[ref])) for ref in changed_refs]
old = {}
for r in old_refs:
old_ref = self.map_hg_get(old_refs[r])
if old_ref:
old[bin(old_ref)] = 1
return old, new
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
def get_refs(self, remote):
self.export_commits()
client, path = self.get_transport_and_path(remote)
old_refs = {}
new_refs = {}
def changed(refs):
old_refs.update(refs)
to_push = set(self.local_heads().values() + self.tags.values())
new_refs.update(self.get_changed_refs(refs, to_push, True))
# don't push anything
return {}
try:
client.send_pack(path, changed, lambda have, want: [])
changed_refs = [ref for ref, sha in new_refs.iteritems()
if sha != old_refs.get(ref)]
new = [bin(self.map_hg_get(new_refs[ref])) for ref in changed_refs]
old = {}
for r in old_refs:
old_ref = self.map_hg_get(old_refs[r])
if old_ref:
old[bin(old_ref)] = 1
return old, new
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
def get_refs(self, remote):
self.export_commits()
client, path = self.get_transport_and_path(remote)
old_refs = {}
new_refs = {}
def changed(refs):
old_refs.update(refs)
to_push = set(self.local_heads().values() + self.tags.values())
new_refs.update(self.get_changed_refs(refs, to_push, True))
# don't push anything
return {}
try:
client.send_pack(path, changed, None)
changed_refs = [
ref for ref, sha in new_refs.iteritems()
if sha != old_refs.get(ref)
]
new = [bin(self.map_hg_get(new_refs[ref])) for ref in changed_refs]
old = dict((bin(self.map_hg_get(old_refs[r])), 1)
for r in changed_refs if r in old_refs)
return old, new
except HangupException:
raise hgutil.Abort("the remote end hung up unexpectedly")
def push(self, progress_func, username=None, password=None):
""" This pushes updates to a remote repository.
This code has been take from dulwich.porcelain.
"""
if not self.remote_url:
return
logger.info("Pushing to '%s' ..." % (self.remote_url))
# Get the client and path
client, path = get_transport_and_path_from_url(self.remote_url)
if password:
client.ssh_vendor.ssh_kwargs["password"] = password
selected_refs = []
refspecs = b"+refs/heads/master"
def update_refs(refs):
selected_refs.extend(
parse_reftuples(self.git_repository.refs, refs, refspecs))
new_refs = {}
# TODO: Handle selected_refs == {None: None}
for (lh, rh, force) in selected_refs:
if lh is None:
new_refs[rh] = ZERO_SHA
else:
new_refs[rh] = self.git_repository.refs[lh]
return new_refs
try:
client.send_pack(
path.encode('utf-8'),
update_refs,
self.git_repository.object_store.generate_pack_contents,
progress=progress_func)
progress_func(b"Push successful.\n")
except (UpdateRefsError, SendPackError) as e:
progress_func(b"Push failed -> " + e.message.encode('utf8') +
b"\n")
def send_and_verify(self, branch, local, target):
"""Send branch from local to remote repository and verify it worked."""
client = LocalGitClient()
ref_name = b"refs/heads/" + branch
new_refs = client.send_pack(target.path,
lambda _: {ref_name: local.refs[ref_name]},
local.object_store.generate_pack_data)
self.assertEqual(local.refs[ref_name], new_refs[ref_name])
obj_local = local.get_object(new_refs[ref_name])
obj_target = target.get_object(new_refs[ref_name])
self.assertEqual(obj_local, obj_target)
def upload_pack(self, remote, revs, force):
client, path = self.get_transport_and_path(remote)
def changed(refs):
to_push = revs or set(self.local_heads().values() + self.tags.values())
return self.get_changed_refs(refs, to_push, force)
genpack = self.git.object_store.generate_pack_contents
try:
self.ui.status(_("creating and sending data\n"))
changed_refs = client.send_pack(path, changed, genpack)
return changed_refs
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
def send_and_verify(self, branch, local, target):
"""Send branch from local to remote repository and verify it worked."""
client = LocalGitClient()
ref_name = b"refs/heads/" + branch
new_refs = client.send_pack(target.path,
lambda _: {ref_name: local.refs[ref_name]},
local.object_store.generate_pack_contents)
self.assertEqual(local.refs[ref_name], new_refs[ref_name])
obj_local = local.get_object(new_refs[ref_name])
obj_target = target.get_object(new_refs[ref_name])
self.assertEqual(obj_local, obj_target)
def upload_pack(self, remote, revs, force):
client, path = self.get_transport_and_path(remote)
def changed(refs):
to_push = revs or set(self.local_heads().values() + self.tags.values())
return self.get_changed_refs(refs, to_push, force)
genpack = self.git.object_store.generate_pack_contents
try:
self.ui.status(_("creating and sending data\n"))
changed_refs = client.send_pack(path, changed, genpack)
return changed_refs
except HangupException:
raise hgutil.Abort("the remote end hung up unexpectedly")
def send_and_verify(self, branch, local, target):
client = LocalGitClient()
ref_name = b"refs/heads/" + branch
new_refs = client.send_pack(target.path,
lambda _: {ref_name: local.refs[ref_name]},
local.object_store.generate_pack_contents)
self.assertEqual(local.refs[ref_name], new_refs[ref_name])
for name, sha in new_refs.items():
self.assertEqual(new_refs[name], target.refs[name])
obj_local = local.get_object(new_refs[ref_name])
obj_target = target.get_object(new_refs[ref_name])
self.assertEqual(obj_local, obj_target)
def upload_pack(self, remote, revs, force):
client, path = self.get_transport_and_path(remote)
old_refs = {}
def changed(refs):
old_refs.update(refs)
to_push = revs or set(self.local_heads().values() + self.tags.values())
return self.get_changed_refs(refs, to_push, force)
genpack = self.git.object_store.generate_pack_contents
try:
self.ui.status(_("creating and sending data\n"))
new_refs = client.send_pack(path, changed, genpack)
return old_refs, new_refs
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
def send_and_verify(self, branch, local, target):
client = LocalGitClient()
ref_name = "refs/heads/" + branch
new_refs = client.send_pack(target.path,
lambda _: { ref_name: local.refs[ref_name] },
local.object_store.generate_pack_contents)
self.assertEqual(local.refs[ref_name], new_refs[ref_name])
for name, sha in new_refs.iteritems():
self.assertEqual(new_refs[name], target.refs[name])
obj_local = local.get_object(new_refs[ref_name])
obj_target = target.get_object(new_refs[ref_name])
self.assertEqual(obj_local, obj_target)
def get_refs(self, remote):
self.export_commits()
client, path = self.get_transport_and_path(remote)
old_refs = {}
new_refs = {}
def changed(refs):
old_refs.update(refs)
to_push = set(self.local_heads().values() + self.tags.values())
new_refs.update(self.get_changed_refs(refs, to_push, True))
# don't push anything
return {}
try:
client.send_pack(path, changed, None)
changed_refs = [ref for ref, sha in new_refs.iteritems()
if sha != old_refs.get(ref)]
new = [bin(self.map_hg_get(new_refs[ref])) for ref in changed_refs]
old = dict( (bin(self.map_hg_get(old_refs[r])), 1)
for r in changed_refs if r in old_refs)
return old, new
except HangupException:
raise hgutil.Abort("the remote end hung up unexpectedly")
def send_and_verify(self, branch, local, target):
"""Send branch from local to remote repository and verify it worked."""
client = LocalGitClient()
ref_name = b"refs/heads/" + branch
result = client.send_pack(target.path,
lambda _: {ref_name: local.refs[ref_name]},
local.generate_pack_data)
self.assertEqual(local.refs[ref_name], result.refs[ref_name])
self.assertIs(None, result.agent)
self.assertEqual({}, result.ref_status)
obj_local = local.get_object(result.refs[ref_name])
obj_target = target.get_object(result.refs[ref_name])
self.assertEqual(obj_local, obj_target)
def create_remote_refs(git_url, ref_mutator):
"""Creates refs (tags, branches) on a remote git repo.
:param git_url: the URL or path to the remote git repo.
:param ref_mutator: A function that determines the new refs to create on
the remote repo. This gets passed a dictionary of the
remote server's refs in the format {name : hash, ...},
and should return a dictionary of the same format.
:returns: The map of refs, with our changes applied.
"""
client, path = dulwich.client.get_transport_and_path(git_url)
determine_wants = _make_determine_wants_func(ref_mutator)
# We know we don't need to push any objects.
generate_pack_contents = lambda have, want: []
return client.send_pack(path, determine_wants, generate_pack_contents)
def upload_pack(self, remote, revs, force):
client, path = self.get_transport_and_path(remote)
old_refs = {}
change_totals = {}
def changed(refs):
self.ui.status(_("searching for changes\n"))
old_refs.update(refs)
to_push = revs or set(self.local_heads().values() + self.tags.values())
return self.get_changed_refs(refs, to_push, force)
def genpack(have, want):
commits = []
for mo in self.git.object_store.find_missing_objects(have, want):
(sha, name) = mo
o = self.git.object_store[sha]
t = type(o)
change_totals[t] = change_totals.get(t, 0) + 1
if isinstance(o, Commit):
commits.append(sha)
commit_count = len(commits)
self.ui.note(_("%d commits found\n") % commit_count)
if commit_count > 0:
self.ui.debug(_("list of commits:\n"))
for commit in commits:
self.ui.debug("%s\n" % commit)
self.ui.status(_("adding objects\n"))
return self.git.object_store.generate_pack_contents(have, want)
try:
new_refs = client.send_pack(path, changed, genpack)
if len(change_totals) > 0:
self.ui.status(_("added %d commits with %d trees"
" and %d blobs\n") %
(change_totals.get(Commit, 0),
change_totals.get(Tree, 0),
change_totals.get(Blob, 0)))
return old_refs, new_refs
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
def create_remote_refs(git_url, ref_mutator, force=False):
"""Creates refs (tags, branches) on a remote git repo.
:param git_url: the URL or path to the remote git repo.
:param ref_mutator: A function that determines the new refs to create on
the remote repo. This gets passed a dictionary of the
remote server's refs in the format {name : hash, ...},
and should return a dictionary of the same format.
:param force: Bool, defaults to false. If true we will overwrite
refs even if they are already set.
:returns: The map of refs, with our changes applied.
"""
client, path = dulwich.client.get_transport_and_path(git_url)
if force is False:
determine_wants = _make_determine_wants_func(ref_mutator)
else:
determine_wants = ref_mutator
# We know we don't need to push any objects.
def generate_pack_contents(have, want):
return []
return client.send_pack(path, determine_wants, generate_pack_contents)
def create_remote_refs(git_url, ref_mutator, force=False):
"""Creates refs (tags, branches) on a remote git repo.
:param git_url: the URL or path to the remote git repo.
:param ref_mutator: A function that determines the new refs to create on
the remote repo. This gets passed a dictionary of the
remote server's refs in the format {name : hash, ...},
and should return a dictionary of the same format.
:param force: Bool, defaults to false. If true we will overwrite
refs even if they are already set.
:returns: The map of refs, with our changes applied.
"""
client, path = dulwich.client.get_transport_and_path(git_url)
if force is False:
determine_wants = _make_determine_wants_func(ref_mutator)
else:
determine_wants = ref_mutator
# We know we don't need to push any objects.
def generate_pack_contents(have, want):
return []
return client.send_pack(path, determine_wants, generate_pack_contents)
def push_with_key(repo_path,
remote="ssh://[email protected]:22/QGB/QPSU.git",
private_key='',
refspecs='master',
errstream=getattr(sys.stderr, 'buffer', None),
private_key_password=None,
retry=3):
r''' #important# 三引号string中不能出现 \util 这种字符（常见于路径）
# 会导致 SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 1-2: truncated \uXXXX escape 错误
# 最好 引号前加r 强制用 raw-string
push ... 2021-04-15__06.26.44__.288
File "C:/QGB/babun/cygwin/bin\qgb\git.py", line 106, in up
push_with_key(repo.path,url,**ka)
File "C:/QGB/babun/cygwin/bin\qgb\git.py", line 205, in push_with_key
progress=errstream.write)
File "C:\QGB\Anaconda3\lib\site-packages\dulwich\client.py", line 789, in send_pack
proto, unused_can_read, stderr = self._connect(b'receive-pack', path)
File "C:\QGB\Anaconda3\lib\site-packages\dulwich\client.py", line 1410, in _connect
**kwargs)
File "C:\QGB\Anaconda3\lib\site-packages\dulwich\contrib\paramiko_vendor.py", line 103, in run_command
client.connect(**connection_kwargs)
File "C:\QGB\Anaconda3\lib\site-packages\paramiko\client.py", line 349, in connect
retry_on_signal(lambda: sock.connect(addr))
File "C:\QGB\Anaconda3\lib\site-packages\paramiko\util.py", line 283, in retry_on_signal
return function()
File "C:\QGB\Anaconda3\lib\site-packages\paramiko\client.py", line 349, in <lambda>
retry_on_signal(lambda: sock.connect(addr))
Out[26]: 'C:/QGB/babun/cygwin/bin/qgb/'
'''
from dulwich.contrib.paramiko_vendor import ParamikoSSHVendor
import dulwich.porcelain
from dulwich.protocol import (
Protocol,
ZERO_SHA,
)
from dulwich.objectspec import (
parse_commit,
parse_object,
parse_ref,
parse_reftuples,
parse_tree,
)
from dulwich.errors import (
GitProtocolError,
NotGitRepository,
SendPackError,
UpdateRefsError,
)
DEFAULT_ENCODING = 'utf-8'
selected_refs = []
if not private_key:
private_key = U.get_or_set('id_rsa', F.get_home() + '.ssh/id_rsa')
private_key = U.set('id_rsa', private_key)
with open_private_key(
private_key) as fpkey, dulwich.porcelain.open_repo_closing(
repo_path) as r:
def update_refs(refs):
selected_refs.extend(parse_reftuples(r.refs, refs, refspecs))
new_refs = {}
# TODO: Handle selected_refs == {None: None}
for (lh, rh, force) in selected_refs:
if lh is None:
new_refs[rh] = ZERO_SHA
else:
new_refs[rh] = r.refs[lh]
return new_refs
import paramiko
pkey = paramiko.RSAKey.from_private_key(fpkey,
password=private_key_password)
if 'git@' not in repo_path:
# raise py.NotImplementedError("'git@' not in repo_path")
repo_path
client, path = dulwich.client.get_transport_and_path(
remote, vendor=ParamikoSSHVendor(pkey=pkey))
err_encoding = getattr(errstream, 'encoding', None) or DEFAULT_ENCODING
remote_location_bytes = client.get_url(path).encode(err_encoding)
#b'ssh://[email protected]:22/QGB/QPSU.git'
try:
client.send_pack(
path,
update_refs,
generate_pack_data=r.object_store.generate_pack_data,
progress=errstream.write)
# print(remote_location_bytes)
errstream.write(b"Push to " + remote_location_bytes +
b" successful.\n")
errstream.flush() # 与 stdout 混合打印时 ，顺序可能不同
except (UpdateRefsError, SendPackError) as e:
errstream.write(b"Push to " + remote_location_bytes +
b" failed -> " + e.message.encode(err_encoding) +
b"\n")
return py.No(e)
finally:
errstream.flush()
return client
def push_refspec(
self,
url: str,
src: Optional[str],
dest: str,
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
progress: Callable[["GitProgressEvent"], None] = None,
**kwargs,
):
from dulwich.client import HTTPUnauthorized, get_transport_and_path
from dulwich.errors import NotGitRepository, SendPackError
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
dest_refs, values = self._push_dest_refs(src, dest)
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
def update_refs(refs):
from dulwich.objects import ZERO_SHA
new_refs = {}
for ref, value in zip(dest_refs, values):
if ref in refs and value != ZERO_SHA:
local_sha = self.repo.refs[ref]
remote_sha = refs[ref]
try:
check_diverged(self.repo, remote_sha, local_sha)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(ref), os.fsdecode(remote_sha))
if not overwrite:
continue
new_refs[ref] = value
return new_refs
try:
from dvc.scm.progress import GitProgressReporter
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=GitProgressReporter(progress) if progress else None,
)
except (NotGitRepository, SendPackError) as exc:
raise SCMError("Git failed to push '{src}' to '{url}'") from exc
except HTTPUnauthorized:
raise AuthError(url)
def push_refspec(
self,
url: str,
src: Optional[str],
dest: str,
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
**kwargs,
):
from dulwich.client import HTTPUnauthorized, get_transport_and_path
from dulwich.errors import NotGitRepository, SendPackError
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
dest_refs, values = self._push_dest_refs(src, dest)
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
def update_refs(refs):
from dulwich.objects import ZERO_SHA
new_refs = {}
for ref, value in zip(dest_refs, values):
if ref in refs and value != ZERO_SHA:
local_sha = self.repo.refs[ref]
remote_sha = refs[ref]
try:
check_diverged(self.repo, remote_sha, local_sha)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(ref), os.fsdecode(remote_sha))
if not overwrite:
continue
new_refs[ref] = value
return new_refs
try:
with Tqdm(desc="Pushing git refs",
bar_format=self.BAR_FMT_NOTOTAL) as pbar:
def progress(msg_b):
msg = msg_b.decode("ascii").strip()
pbar.update_msg(msg)
pbar.refresh()
logger.trace(msg)
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=progress,
)
except (NotGitRepository, SendPackError) as exc:
raise SCMError("Git failed to push '{src}' to '{url}'") from exc
except HTTPUnauthorized:
raise GitAuthError(url)
