def _fetch_pack(self):
"""Fetch changes and store them in a pack."""
def prepare_refs(refs):
return [ref.hash.encode('utf-8') for ref in refs
if not ref.refname.endswith('^{}')]
def determine_wants(refs):
remote_refs = prepare_refs(self._discover_refs(remote=True))
local_refs = prepare_refs(self._discover_refs())
wants = [ref for ref in remote_refs if ref not in local_refs]
return wants
client, repo_path = dulwich.client.get_transport_and_path(self.uri)
repo = dulwich.repo.Repo(self.dirpath)
fd = io.BytesIO()
local_refs = self._discover_refs()
graph_walker = _GraphWalker(local_refs)
result = client.fetch_pack(repo_path,
determine_wants,
graph_walker,
fd.write)
refs = [GitRef(ref_hash.decode('utf-8'), ref_name.decode('utf-8'))
for ref_name, ref_hash in result.refs.items()]
if len(fd.getvalue()) > 0:
fd.seek(0)
pack = repo.object_store.add_thin_pack(fd.read, None)
pack_name = pack.name().decode('utf-8')
else:
pack_name = None
return (pack_name, refs)
def fetch_pack(self, remote_name, heads):
client, path = self.get_transport_and_path(remote_name)
graphwalker = self.git.get_graph_walker()
def determine_wants(refs):
if heads:
want = []
# contains pairs of ('refs/(heads|tags|...)/foo', 'foo')
# if ref is just '<foo>', then we get ('foo', 'foo')
stripped_refs = [
(r, r[r.find('/', r.find('/')+1)+1:])
for r in refs]
for h in heads:
r = [pair[0] for pair in stripped_refs if pair[1] == h]
if not r:
raise hgutil.Abort("ref %s not found on remote server" % h)
elif len(r) == 1:
want.append(refs[r[0]])
else:
raise hgutil.Abort("ambiguous reference %s: %r" % (h, r))
else:
want = [sha for ref, sha in refs.iteritems()
if not ref.endswith('^{}')]
return want
f, commit = self.git.object_store.add_pack()
try:
try:
return client.fetch_pack(path, determine_wants, graphwalker,
f.write, self.ui.status)
except HangupException:
raise hgutil.Abort("the remote end hung up unexpectedly")
finally:
commit()
def fetch_pack(self, remote_name, heads):
client, path = self.get_transport_and_path(remote_name)
graphwalker = self.git.get_graph_walker()
def determine_wants(refs):
if heads:
want = []
for h in heads:
r = [ref for ref in refs if ref.endswith('/'+h)]
if not r:
raise hgutil.Abort("ref %s not found on remote server")
elif len(r) == 1:
want.append(refs[r[0]])
else:
raise hgutil.Abort("ambiguous reference %s: %r"%(h, r))
else:
want = [sha for ref, sha in refs.iteritems()
if not ref.endswith('^{}')]
return want
f, commit = self.git.object_store.add_pack()
try:
try:
return client.fetch_pack(path, determine_wants, graphwalker, f.write, self.ui.status)
except HangupException:
raise hgutil.Abort("the remote end hung up unexpectedly")
finally:
commit()
def list_remote_refs(git_url):
"""Get the refs from a remote git repo as a dictionary of name->hash."""
client, path = dulwich.client.get_transport_and_path(git_url)
try:
return client.fetch_pack(path, lambda refs: [], None, None)
except dulwich.errors.HangupException as e:
raise LSRemoteException("Unable to fetch remote refs: %s" % e)
def list_remote_refs(git_url):
"""Get the refs from a remote git repo as a dictionary of name->hash."""
client, path = dulwich.client.get_transport_and_path(git_url)
try:
refs = client.fetch_pack(path, lambda refs: [], None, lambda data: None)
return {k.decode('UTF-8'): v.decode('UTF-8') for k, v in refs.items()}
except dulwich.errors.HangupException as e:
raise LSRemoteException("Unable to fetch remote refs: %s" % e)
def fetch_pack(self, remote_name, heads):
client, path = self.get_transport_and_path(remote_name)
graphwalker = self.git.get_graph_walker()
def determine_wants(refs):
if heads:
want = []
# contains pairs of ('refs/(heads|tags|...)/foo', 'foo')
# if ref is just '<foo>', then we get ('foo', 'foo')
stripped_refs = [(r, r[r.find('/',
r.find('/') + 1) + 1:])
for r in refs]
for h in heads:
r = [pair[0] for pair in stripped_refs if pair[1] == h]
if not r:
raise hgutil.Abort(
"ref %s not found on remote server" % h)
elif len(r) == 1:
want.append(refs[r[0]])
else:
raise hgutil.Abort(
"ambiguous reference %s: %r" % (h, r))
else:
want = [
sha for ref, sha in refs.iteritems()
if not ref.endswith('^{}') and (ref.startswith(
'refs/heads/') or ref.startswith('refs/tags/'))
]
want = [x for x in want if x not in self.git]
return want
f, commit = self.git.object_store.add_pack()
try:
try:
progress = GitProgress(self.ui)
ret = client.fetch_pack(path, determine_wants, graphwalker,
f.write, progress.progress)
progress.flush()
return ret
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
finally:
commit()
def fetch_pack(self, remote_name, heads):
client, path = self.get_transport_and_path(remote_name)
graphwalker = self.git.get_graph_walker()
def determine_wants(refs):
if heads:
want = []
# contains pairs of ('refs/(heads|tags|...)/foo', 'foo')
# if ref is just '<foo>', then we get ('foo', 'foo')
stripped_refs = [(r, r[r.find('/',
r.find('/') + 1) + 1:])
for r in refs]
for h in heads:
r = [pair[0] for pair in stripped_refs if pair[1] == h]
if not r:
raise hgutil.Abort(
"ref %s not found on remote server" % h)
elif len(r) == 1:
want.append(refs[r[0]])
else:
raise hgutil.Abort("ambiguous reference %s: %r" %
(h, r))
else:
want = [
sha for ref, sha in refs.iteritems()
if not ref.endswith('^{}') and (ref.startswith(
'refs/heads/') or ref.startswith('refs/tags/'))
]
want = [x for x in want if x not in self.git]
return want
f, commit = self.git.object_store.add_pack()
try:
try:
progress = GitProgress(self.ui)
ret = client.fetch_pack(path, determine_wants, graphwalker,
f.write, progress.progress)
progress.flush()
return ret
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
finally:
commit()
def fetch_pack(self, remote_name, heads):
client, path = self.get_transport_and_path(remote_name)
graphwalker = self.git.get_graph_walker()
def determine_wants(refs):
if heads:
want = []
# contains pairs of ('refs/(heads|tags|...)/foo', 'foo')
# if ref is just '<foo>', then we get ('foo', 'foo')
stripped_refs = [(r, r[r.find('/',
r.find('/') + 1) + 1:])
for r in refs]
for h in heads:
r = [pair[0] for pair in stripped_refs if pair[1] == h]
if not r:
raise hgutil.Abort(
"ref %s not found on remote server" % h)
elif len(r) == 1:
want.append(refs[r[0]])
else:
raise hgutil.Abort("ambiguous reference %s: %r" %
(h, r))
else:
want = [
sha for ref, sha in refs.iteritems()
if not ref.endswith('^{}')
]
return want
f, commit = self.git.object_store.add_pack()
try:
try:
return client.fetch_pack(path, determine_wants, graphwalker,
f.write, self.ui.status)
except HangupException:
raise hgutil.Abort("the remote end hung up unexpectedly")
finally:
commit()
def fetch_pack_from_origin(
self,
origin_url: str,
base_repo: RepoRepresentation,
do_activity: Callable[[bytes], None],
) -> FetchPackReturn:
"""Fetch a pack from the origin"""
pack_buffer = SpooledTemporaryFile(max_size=self.temp_file_cutoff)
transport_url = origin_url
logger.debug("Transport url to communicate with server: %s", transport_url)
client, path = dulwich.client.get_transport_and_path(
transport_url, thin_packs=False
)
logger.debug("Client %s to fetch pack at %s", client, path)
size_limit = self.pack_size_bytes
def do_pack(data: bytes) -> None:
cur_size = pack_buffer.tell()
would_write = len(data)
if cur_size + would_write > size_limit:
raise IOError(
f"Pack file too big for repository {origin_url}, "
f"limit is {size_limit} bytes, current size is {cur_size}, "
f"would write {would_write}"
)
pack_buffer.write(data)
pack_result = client.fetch_pack(
path,
base_repo.determine_wants,
base_repo.graph_walker(),
do_pack,
progress=do_activity,
)
remote_refs = pack_result.refs or {}
symbolic_refs = pack_result.symrefs or {}
pack_buffer.flush()
pack_size = pack_buffer.tell()
pack_buffer.seek(0)
logger.debug("fetched_pack_size=%s", pack_size)
# check if repository only supports git dumb transfer protocol,
# fetched pack file will be empty in that case as dulwich do
# not support it and do not fetch any refs
self.dumb = transport_url.startswith("http") and getattr(client, "dumb", False)
return FetchPackReturn(
remote_refs=utils.filter_refs(remote_refs),
symbolic_refs=utils.filter_refs(symbolic_refs),
pack_buffer=pack_buffer,
pack_size=pack_size,
)