def receivepack(ui, fh, dpack, hpack, version=1):
# type: (UI, IO[bytes], mutabledatastore, mutablehistorystore, int) -> Tuple[List[Tuple[bytes, bytes]], List[Tuple[bytes, bytes]]]
receiveddata = []
receivedhistory = []
size = 0
with progress.bar(ui, _("receiving pack")) as prog:
while True:
filename = readpath(fh)
count = 0
# Store the history for later sorting
for value in readhistory(fh):
node, p1, p2, linknode, copyfrom = value
hpack.add(filename, node, p1, p2, linknode, copyfrom)
receivedhistory.append((filename, node))
count += 1
size += len(filename) + len(node) + sum(
len(x or "") for x in value)
for node, deltabase, delta, metadata in readdeltas(
fh, version=version):
dpack.add(filename, node, deltabase, delta, metadata=metadata)
receiveddata.append((filename, node))
count += 1
size += len(filename) + len(node) + len(deltabase) + len(delta)
if count == 0 and filename == "":
break
prog.value += 1
perftrace.tracebytes("Received Pack Size", size)
return receiveddata, receivedhistory
def receivepack(
ui: "UI",
fh: "IO[bytes]",
dpack: "mutabledatastore",
hpack: "mutablehistorystore",
version: int = 1,
) -> "Tuple[List[Tuple[bytes, bytes]], List[Tuple[bytes, bytes]]]":
receiveddata = []
receivedhistory = []
size = 0
start = time.time()
with progress.bar(ui, _("receiving pack")) as prog:
while True:
filename = readpath(fh)
count = 0
# Store the history for later sorting
for value in readhistory(fh):
node, p1, p2, linknode, copyfrom = value
hpack.add(filename, node, p1, p2, linknode, copyfrom)
receivedhistory.append((filename, node))
count += 1
size += len(filename) + len(node) + sum(
len(x or "") for x in value)
for node, deltabase, delta, metadata in readdeltas(
fh, version=version):
dpack.add(filename, node, deltabase, delta, metadata=metadata)
receiveddata.append((filename, node))
count += 1
size += len(filename) + len(node) + len(deltabase) + len(delta)
if count == 0 and filename == "":
break
prog.value += 1
perftrace.tracebytes("Received Pack Size", size)
duration = time.time() - start
megabytes = float(size) / 1024 / 1024
if ui.configbool("remotefilelog", "debug-fetches") and (
duration > 1 or len(receiveddata) > 100 or megabytes > 1):
ui.warn(
_("Receive pack: %s entries, %.2f MB, %.2f seconds (%0.2f MBps)\n")
% (len(receiveddata), megabytes, duration, megabytes / duration))
return receiveddata, receivedhistory
def _batch(self, pointers, localstore, action, objectnames=None):
if action not in ["upload", "download"]:
raise error.ProgrammingError("invalid Git-LFS action: %s" % action)
response = self._batchrequest(pointers, action)
objects = self._extractobjects(response, pointers, action)
total = sum(x.get("size", 0) for x in objects)
perftrace.tracebytes("Size", total)
sizes = {}
for obj in objects:
sizes[obj.get("oid")] = obj.get("size", 0)
topic = {
"upload": _("lfs uploading"),
"download": _("lfs downloading")
}[action]
if self.ui.verbose and len(objects) > 1:
self.ui.write(
_("lfs: need to transfer %d objects (%s)\n") %
(len(objects), util.bytecount(total)))
def transfer(chunk):
for obj in chunk:
objsize = obj.get("size", 0)
if self.ui.verbose:
if action == "download":
msg = _("lfs: downloading %s (%s)\n")
elif action == "upload":
msg = _("lfs: uploading %s (%s)\n")
self.ui.write(msg %
(obj.get("oid"), util.bytecount(objsize)))
retry = self.retry
while True:
try:
yield 0, obj.get("oid")
self._basictransfer(obj, action, localstore)
yield 1, obj.get("oid")
break
except Exception as ex:
if retry > 0:
if self.ui.verbose:
self.ui.write(
_("lfs: failed: %r (remaining retry %d)\n")
% (ex, retry))
retry -= 1
continue
raise
starttimestamp = util.timer()
if action == "download":
oids = worker.worker(
self.ui,
0.1,
transfer,
(),
sorted(objects, key=lambda o: o.get("oid")),
preferthreads=True,
callsite="blobstore",
)
else:
oids = transfer(objects)
transferred = 0
with progress.bar(self.ui,
topic,
_("bytes"),
total=total,
formatfunc=util.bytecount) as prog:
for count, oid in oids:
if count != 0:
transferred += sizes[oid]
if self.ui.verbose:
self.ui.write(_("lfs: processed: %s\n") % oid)
if objectnames is not None:
prog.value = (transferred, objectnames.get(oid, ""))
else:
prog.value = transferred
currenttimestamp = util.timer()
self._metrics["lfs_%s_size" % action] += total
self._metrics["lfs_%s_time" % action] += (currenttimestamp - max(
self._timestamp["latest_%s_timestamp" % action],
starttimestamp)) * 1000
self._timestamp["latest_%s_timestamp" % action] = currenttimestamp