def progresstest(ui, loops, total, **opts):
loops = int(loops)
total = int(total)
if total == -1:
total = None
nested = opts.get("nested", None)
useunicode = opts.get("unicode", False)
if useunicode:
topic = pycompat.ensurestr("\u3042\u3044\u3046\u3048")
else:
topic = "progress test"
with progress.bar(ui, topic, "cycles", total) as prog:
for i in range(loops + 1):
if useunicode:
prog.value = (i, unicodeloopitems[i % len(unicodeloopitems)])
else:
prog.value = (i, "loop %s" % i)
syncrender()
if nested:
nestedtotal = 5 if i % 6 == 5 else 2
with progress.bar(ui, "nested progress",
total=nestedtotal) as nestedprog:
for j in range(nestedtotal + 1):
nestedprog.value = (j, "nest %s" % j)
syncrender()
def progresstest(ui, loops, total, **opts):
loops = int(loops)
total = int(total)
if total == -1:
total = None
nested = opts.get("nested", None)
useunicode = opts.get("unicode", False)
if useunicode:
topic = u"\u3042\u3044\u3046\u3048".encode("utf-8")
else:
topic = "progress test"
with progress.bar(ui, topic, "cycles", total) as prog:
for i in range(loops + 1):
if useunicode:
prog.value = (i, unicodeloopitems[i % len(unicodeloopitems)])
else:
prog.value = (i, "loop %s" % i)
progress._engine.pump(_faketime.increment())
if nested:
nestedtotal = 5 if i % 6 == 5 else 2
with progress.bar(ui, "nested progress",
total=nestedtotal) as nestedprog:
for j in range(nestedtotal + 1):
nestedprog.value = (j, "nest %s" % j)
progress._engine.pump(_faketime.increment())
def receivepack(ui, fh, dpack, hpack, version=1):
# type: (UI, IO[bytes], mutabledatastore, mutablehistorystore, int) -> Tuple[List[Tuple[bytes, bytes]], List[Tuple[bytes, bytes]]]
receiveddata = []
receivedhistory = []
size = 0
with progress.bar(ui, _("receiving pack")) as prog:
while True:
filename = readpath(fh)
count = 0
# Store the history for later sorting
for value in readhistory(fh):
node, p1, p2, linknode, copyfrom = value
hpack.add(filename, node, p1, p2, linknode, copyfrom)
receivedhistory.append((filename, node))
count += 1
size += len(filename) + len(node) + sum(
len(x or "") for x in value)
for node, deltabase, delta, metadata in readdeltas(
fh, version=version):
dpack.add(filename, node, deltabase, delta, metadata=metadata)
receiveddata.append((filename, node))
count += 1
size += len(filename) + len(node) + len(deltabase) + len(delta)
if count == 0 and filename == "":
break
prog.value += 1
perftrace.tracebytes("Received Pack Size", size)
return receiveddata, receivedhistory
def downloadtest(description, bytecount):
pipeo.write(b"download %i\n" % bytecount)
pipeo.flush()
l = pipei.readline()
if not l or not l.startswith(b"download bytes"):
raise error.Abort("invalid response from server: %r" % l)
bytecount = int(l.split()[2])
with progress.bar(ui,
description,
total=bytecount,
formatfunc=util.bytecount) as prog:
starttime = util.timer()
remaining = bytecount
while remaining > 0:
data = pipei.read(min(remaining, BLOCK_SIZE))
if not data:
raise error.Abort(
"premature end of speed-test download stream")
remaining -= len(data)
prog.value = bytecount - remaining
l = pipei.readline()
if not l or not l.startswith(b"download complete"):
raise error.Abort("invalid response from server: %r" % l)
endtime = util.timer()
return endtime - starttime
def _marklookupsclean(self):
dirstate = self.dirstate
normal = dirstate.normal
newdmap = dirstate._map
cleanlookups = self.cleanlookups
self.cleanlookups = []
repo = dirstate._repo
p1 = dirstate.p1()
wctx = repo[None]
pctx = repo[p1]
with progress.bar(self.ui, _("marking clean"), _("files"),
len(cleanlookups)) as prog:
count = 0
for f in cleanlookups:
count += 1
prog.value = (count, f)
# Only make something clean if it's already in a
# normal state. Things in other states, like 'm'
# merge state, should not be marked clean.
entry = newdmap[f]
if entry[0] == "n" and f not in newdmap.copymap and entry[
2] != -2:
# It may have been a while since we added the
# file to cleanlookups, so double check that
# it's still clean.
if self._compareondisk(f, wctx, pctx) is False:
normal(f)
def _prefetch(self, revs, base=None, matcher=None):
mfl = self.manifestlog
# Copy the skip set to start large and avoid constant resizing,
# and since it's likely to be very similar to the prefetch set.
files = set()
basemf = self[base or nullid].manifest()
with progress.bar(self.ui, _("prefetching"),
total=len(revs)) as prog:
for rev in sorted(revs):
ctx = self[rev]
if matcher is None:
matcher = self.maybesparsematch(rev)
mfctx = ctx.manifestctx()
mf = mfctx.read()
for path, (new, old) in mf.diff(basemf, matcher).items():
if new[0]:
files.add((path, new[0]))
prog.value += 1
if files:
results = [(path, hex(fnode)) for (path, fnode) in files]
self.fileservice.prefetch(results)
def walktree(self, heads):
"""Return a mapping that identifies the uncommitted parents of every
uncommitted changeset."""
visit = heads
known = set()
parents = {}
numcommits = self.source.numcommits()
with progress.bar(self.ui, _("scanning"), _("revisions"),
numcommits) as prog:
while visit:
n = visit.pop(0)
if n in known:
continue
if n in self.map:
m = self.map[n]
if m == SKIPREV or self.dest.hascommitfrommap(m):
continue
known.add(n)
prog.value = len(known)
commit = self.cachecommit(n)
parents[n] = []
for p in commit.parents:
parents[n].append(p)
visit.append(p)
return parents
def debuggettrees(ui, repo, **opts):
edenapi.bailifdisabled(ui)
keys = []
for line in sys.stdin.readlines():
parts = line.split()
(node, path) = parts if len(parts) > 1 else (parts[0], "")
keys.append((path, node))
dpack, __ = repo.manifestlog.getmutablesharedpacks()
msg = _("fetching %d trees") % len(keys)
with progress.bar(
ui, msg, start=0, unit=_("bytes"), formatfunc=util.bytecount
) as prog:
def progcb(dl, dlt, ul, ult):
if dl > 0:
prog._total = dlt
prog.value = dl
stats = repo.edenapi.get_trees(keys, dpack, progcb)
ui.write(stats.to_str() + "\n")
packpath, __ = repo.manifestlog._mutablesharedpacks.commit()
ui.write(_("wrote datapack: %s\n") % packpath)
def debuggethistory(ui, repo, **opts):
edenapi.bailifdisabled(ui)
input = (line.split() for line in sys.stdin.readlines())
keys = [(path, node) for node, path in input]
depth = opts.get("depth") or None
__, hpack = repo.fileslog.getmutablesharedpacks()
msg = _("fetching history for %d files") % len(keys)
with progress.bar(
ui, msg, start=0, unit=_("bytes"), formatfunc=util.bytecount
) as prog:
def progcb(dl, dlt, ul, ult):
if dl > 0:
prog._total = dlt
prog.value = dl
stats = repo.edenapi.get_history(keys, hpack, depth, progcb)
ui.write(stats.to_str() + "\n")
__, packpath = repo.fileslog._mutablesharedpacks.commit()
ui.write(_("wrote historypack: %s\n") % packpath)
def updateglobalrevmeta(ui, repo, *args, **opts):
"""Reads globalrevs from the latest hg commits and adds them to the
globalrev-hg mapping."""
with repo.wlock(), repo.lock():
unfi = repo
clnode = unfi.changelog.node
clrevision = unfi.changelog.changelogrevision
globalrevmap = _globalrevmap(unfi)
lastrev = globalrevmap.lastrev
repolen = len(unfi)
with progress.bar(ui, _("indexing"), _("revs"), repolen - lastrev) as prog:
def addtoglobalrevmap(grev, node):
if grev:
globalrevmap.add(int(grev), node)
for rev in range(lastrev, repolen): # noqa: F821
hgnode = clnode(rev)
commitdata = clrevision(rev)
extra = commitdata.extra
svnrev = _getsvnrev(extra)
addtoglobalrevmap(svnrev, hgnode)
globalrev = _getglobalrev(ui, extra)
if globalrev != svnrev:
addtoglobalrevmap(globalrev, hgnode)
prog.value += 1
globalrevmap.lastrev = repolen
globalrevmap.save()
def progresstest(ui, total, **opts):
total = int(total)
waitforfile(opts.get("waitfile"))
with progress.bar(ui, "eating", "apples", total) as bar:
for i in range(1, total + 1):
bar.value = i
waitforfile(opts.get("waitfile"))
def debugverifylinkrevcache(ui, repo, *pats, **opts):
"""read the linkrevs from the database and verify if they are correct"""
# restore to the original _adjustlinkrev implementation
c = context.basefilectx
extensions.unwrapfunction(c, "_adjustlinkrev", _adjustlinkrev)
paths = {} # {id: name}
nodes = {} # {id: name}
repo = repo.unfiltered()
idx = repo.unfiltered().changelog.index
db = repo._linkrevcache
paths = dict(db._getdb(db._pathdbname))
nodes = dict(db._getdb(db._nodedbname))
pathsrev = dict(
(v, pycompat.decodeutf8(k)) for k, v in pycompat.iteritems(paths))
nodesrev = dict((v, k) for k, v in pycompat.iteritems(nodes))
lrevs = dict(db._getdb(db._linkrevdbname))
readfilelog = ui.configbool("linkrevcache", "readfilelog", True)
total = len(lrevs)
with progress.bar(ui, _("verifying"), total=total) as prog:
for i, (k, v) in enumerate(pycompat.iteritems(lrevs)):
prog.value = i
pathid, nodeid = k.split(b"\0")
path = pathsrev[pathid]
fnode = nodesrev[nodeid]
linkrevs = _str2intlist(pycompat.decodeutf8(v))
linkrevs.sort()
for linkrev in linkrevs:
fctx = repo[linkrev][path]
introrev = fctx.introrev()
fctx.linkrev()
if readfilelog:
flinkrev = fctx.linkrev()
else:
flinkrev = None
if introrev == linkrev:
continue
if introrev in idx.commonancestorsheads(
introrev, linkrev) and (introrev in linkrevs
or introrev == flinkrev):
adjective = _("unnecessary")
else:
adjective = _("incorrect")
ui.warn(
_("%s linkrev %s for %s @ %s (expected: %s)\n") %
(adjective, linkrev, path, node.hex(fnode), introrev))
ui.write(_("%d entries verified\n") % total)
def countrate(ui, repo, amap, *pats, **opts):
"""Calculate stats"""
opts = pycompat.byteskwargs(opts)
if opts.get("dateformat"):
def getkey(ctx):
t, tz = ctx.date()
date = datetime.datetime(*time.gmtime(float(t) - tz)[:6])
return date.strftime(opts["dateformat"])
else:
tmpl = opts.get("oldtemplate") or opts.get("template")
tmpl = cmdutil.makelogtemplater(ui, repo, tmpl)
def getkey(ctx):
ui.pushbuffer()
tmpl.show(ctx)
return ui.popbuffer()
rate = {}
df = False
if opts.get("date"):
df = util.matchdate(opts["date"])
prog = progress.bar(ui, _("analyzing"), _("revisions"), len(repo))
m = scmutil.match(repo[None], pats, opts)
def prep(ctx, fns):
rev = ctx.rev()
if df and not df(ctx.date()[0]): # doesn't match date format
return
key = getkey(ctx).strip()
key = amap.get(key, key) # alias remap
if opts.get("changesets"):
rate[key] = (rate.get(key, (0, ))[0] + 1, 0)
else:
parents = ctx.parents()
if len(parents) > 1:
ui.note(_("revision %d is a merge, ignoring...\n") % (rev, ))
return
ctx1 = parents[0]
lines = changedlines(ui, repo, ctx1, ctx, fns)
rate[key] = [r + l for r, l in zip(rate.get(key, (0, 0)), lines)]
prog.value += 1
with prog:
for ctx in cmdutil.walkchangerevs(repo, m, opts, prep):
continue
return rate
def debugbuildannotatecache(ui, repo, *pats, **opts):
"""incrementally build fastannotate cache up to REV for specified files
If REV is not specified, use the config 'fastannotate.mainbranch'.
If fastannotate.client is True, download the annotate cache from the
server. Otherwise, build the annotate cache locally.
The annotate cache will be built using the default diff and follow
options and lives in '.hg/fastannotate/default'.
"""
rev = opts.get("REV") or ui.config("fastannotate", "mainbranch")
if not rev:
raise error.Abort(
_("you need to provide a revision"),
hint=_("set fastannotate.mainbranch or use --rev"),
)
if ui.configbool("fastannotate", "unfilteredrepo", True):
repo = repo.unfiltered()
ctx = scmutil.revsingle(repo, rev)
m = scmutil.match(ctx, pats, opts)
paths = list(ctx.walk(m))
if util.safehasattr(repo, "prefetchfastannotate"):
# client
if opts.get("REV"):
raise error.Abort(_("--rev cannot be used for client"))
repo.prefetchfastannotate(paths)
else:
# server, or full repo
with progress.bar(ui, _("building"), total=len(paths)) as prog:
for i, path in enumerate(paths):
prog.value = i
with facontext.annotatecontext(repo, path) as actx:
try:
if actx.isuptodate(rev):
continue
actx.annotate(rev, rev)
except (faerror.CannotReuseError, faerror.CorruptedFileError):
# the cache is broken (could happen with renaming so the
# file history gets invalidated). rebuild and try again.
ui.debug("fastannotate: %s: rebuilding broken cache\n" % path)
actx.rebuild()
try:
actx.annotate(rev, rev)
except Exception as ex:
# possibly a bug, but should not stop us from
# building cache for other files.
ui.warn(
_("fastannotate: %s: failed to " "build cache: %r\n")
% (path, ex)
)
def _processlookups(self, lookups):
repo = self.dirstate._repo
p1 = self.dirstate.p1()
if util.safehasattr(repo, "fileservice"):
p1mf = repo[p1].manifest()
lookupmatcher = matchmod.exact(repo.root, repo.root, lookups)
# We fetch history because we know _compareondisk() uses
# filelog.cmp() which computes the sha(p1, p2, text), which requires
# the history of the file. Later we'll move to comparing content
# hashes, and we can prefetch those hashes instead.
# Note, this may be slow for files with long histories.
repo.fileservice.prefetch(
list((f, hex(p1mf[f])) for f in p1mf.matches(lookupmatcher)),
fetchdata=False,
fetchhistory=True,
)
wctx = repo[None]
pctx = repo[p1]
with progress.bar(self.ui, _("checking changes"), _("files"),
len(lookups)) as prog:
# Sort so we get deterministic ordering. This is important for tests.
count = 0
for fn in sorted(lookups):
count += 1
prog.value = (count, fn)
changed = self._compareondisk(fn, wctx, pctx)
if changed is None:
# File no longer exists
if self.dtolog > 0:
self.dtolog -= 1
self.ui.log("status",
"R %s: checked in filesystem" % fn)
yield (fn, False)
elif changed is True:
# File exists and is modified
if self.mtolog > 0:
self.mtolog -= 1
self.ui.log("status",
"M %s: checked in filesystem" % fn)
yield (fn, True)
else:
# File exists and is clean
if self.ftolog > 0:
self.ftolog -= 1
self.ui.log("status",
"C %s: checked in filesystem" % fn)
self.cleanlookups.append(fn)
def quickchecklog(ui, log, name, knownbroken):
"""
knownbroken: a set of known broken *changelog* revisions
returns (rev, linkrev) of the first bad entry
returns (None, None) if nothing is bad
"""
lookback = 10
rev = max(0, len(log) - lookback)
numchecked = 0
seengood = False
with progress.bar(ui, _("checking %s") % name) as prog:
while rev < len(log):
numchecked += 1
prog.value = (numchecked, rev)
(startflags, clen, ulen, baserev, linkrev, p1, p2,
node) = log.index[rev]
if linkrev in knownbroken:
ui.write(
_("%s: marked corrupted at rev %d (linkrev=%d)\n") %
(name, rev, linkrev))
return rev, linkrev
try:
log.revision(rev, raw=True)
if rev != 0:
if (startflags == 0 or linkrev == 0
or (p1 == 0 and p2 == 0) or clen == 0 or ulen == 0
or node == nullid):
# In theory no 100% correct. But those fields being 0 is
# almost always a corruption practically.
raise ValueError("suspected bad revision data")
seengood = True
rev += 1
except Exception: # RevlogError, mpatchError, ValueError, etc
if rev == 0:
msg = _("all %s entries appear corrupt!") % (name, )
raise error.RevlogError(msg)
if not seengood:
# If the earliest rev we looked at is bad, look back farther
lookback *= 2
rev = max(0, len(log) - lookback)
continue
ui.write(
_("%s: corrupted at rev %d (linkrev=%d)\n") %
(name, rev, linkrev))
return rev, linkrev
ui.write(_("%s looks okay\n") % name)
return None, None
def receivepack(
ui: "UI",
fh: "IO[bytes]",
dpack: "mutabledatastore",
hpack: "mutablehistorystore",
version: int = 1,
) -> "Tuple[List[Tuple[bytes, bytes]], List[Tuple[bytes, bytes]]]":
receiveddata = []
receivedhistory = []
size = 0
start = time.time()
with progress.bar(ui, _("receiving pack")) as prog:
while True:
filename = readpath(fh)
count = 0
# Store the history for later sorting
for value in readhistory(fh):
node, p1, p2, linknode, copyfrom = value
hpack.add(filename, node, p1, p2, linknode, copyfrom)
receivedhistory.append((filename, node))
count += 1
size += len(filename) + len(node) + sum(
len(x or "") for x in value)
for node, deltabase, delta, metadata in readdeltas(
fh, version=version):
dpack.add(filename, node, deltabase, delta, metadata=metadata)
receiveddata.append((filename, node))
count += 1
size += len(filename) + len(node) + len(deltabase) + len(delta)
if count == 0 and filename == "":
break
prog.value += 1
perftrace.tracebytes("Received Pack Size", size)
duration = time.time() - start
megabytes = float(size) / 1024 / 1024
if ui.configbool("remotefilelog", "debug-fetches") and (
duration > 1 or len(receiveddata) > 100 or megabytes > 1):
ui.warn(
_("Receive pack: %s entries, %.2f MB, %.2f seconds (%0.2f MBps)\n")
% (len(receiveddata), megabytes, duration, megabytes / duration))
return receiveddata, receivedhistory
def _pullheadgroups(repo, remotepath, headgroups):
backuplock.progresspulling(
repo, [nodemod.bin(node) for newheads in headgroups for node in newheads]
)
with progress.bar(
repo.ui, _("pulling from commit cloud"), total=len(headgroups)
) as prog:
for index, headgroup in enumerate(headgroups):
headgroupstr = " ".join([head[:12] for head in headgroup])
url = repo.ui.paths.getpath(remotepath).url
repo.ui.status(_("pulling %s from %s\n") % (headgroupstr, url))
prog.value = (index, headgroupstr)
repo.pull(
remotepath,
headnodes=[nodemod.bin(hexnode) for hexnode in headgroup],
quiet=False,
)
repo.connectionpool.close()
def convert(self, sortmode):
try:
self.source.before()
self.dest.before()
self.source.setrevmap(self.map)
self.ui.status(_("scanning source...\n"))
heads = self.source.getheads()
parents = self.walktree(heads)
self.mergesplicemap(parents, self.splicemap)
self.ui.status(_("sorting...\n"))
t = self.toposort(parents, sortmode)
num = len(t)
c = None
self.ui.status(_("converting...\n"))
with progress.bar(self.ui, _("converting"), _("revisions"),
len(t)) as prog:
for i, c in enumerate(t):
num -= 1
desc = self.commitcache[c].desc
if "\n" in desc:
desc = desc.splitlines()[0]
# convert log message to local encoding without using
# tolocal() because the encoding.encoding convert()
# uses is 'utf-8'
self.ui.status("%d %s\n" % (num, recode(desc)))
self.ui.note(_("source: %s\n") % recode(c))
prog.value = i
self.copy(c)
bookmarks = self.source.getbookmarks()
cbookmarks = {}
for k in bookmarks:
v = bookmarks[k]
if self.map.get(v, SKIPREV) != SKIPREV:
cbookmarks[k] = self.map[v]
if c and cbookmarks:
self.dest.putbookmarks(cbookmarks)
self.writeauthormap()
finally:
self.cleanup()
def _httpfetchhistory(self, fileids, hpack, depth=None):
"""Fetch file history over HTTPS using the Eden API"""
n = len(fileids)
msg = (_n(
"fetching history for %d file over HTTPS",
"fetching history for %d files over HTTPS",
n,
) % n)
if self.ui.interactive() and edenapi.debug(self.ui):
self.ui.warn(("%s\n") % msg)
self.ui.metrics.gauge("http_gethistory_revs", n)
self.ui.metrics.gauge("http_gethistory_calls", 1)
with progress.bar(self.ui,
msg,
start=0,
unit=_("bytes"),
formatfunc=util.bytecount) as prog:
def progcallback(dl, dlt, ul, ult):
if dl > 0:
prog._total = dlt
prog.value = dl
stats = self.repo.edenapi.get_history(fileids, hpack, depth,
progcallback)
if self.ui.interactive() and edenapi.debug(self.ui):
self.ui.warn(_("%s\n") % stats.to_str())
self.ui.metrics.gauge("http_gethistory_time_ms",
stats.time_in_millis())
self.ui.metrics.gauge("http_gethistory_latency_ms",
stats.latency_in_millis())
self.ui.metrics.gauge("http_gethistory_bytes_downloaded",
stats.downloaded())
self.ui.metrics.gauge("http_gethistory_bytes_uploaded",
stats.uploaded())
self.ui.metrics.gauge("http_gethistory_requests", stats.requests())
def _linkrev(self):
if self._filenode == nullid:
return nullrev
p1, p2, linknode, copyfrom = self.getnodeinfo()
rev = self._repo.changelog.nodemap.get(linknode)
if rev is not None:
return rev
# Search all commits for the appropriate linkrev (slow, but uncommon)
repo = self._repo
path = self._path
fileid = self._filenode
cl = repo.changelog
mfl = repo.manifestlog
with repo.ui.timesection("scanlinkrev"), repo.ui.configoverride({
("treemanifest", "fetchdepth"):
1
}), perftrace.trace("Scanning for Linkrev"), progress.bar(
repo.ui,
_("scanning for linkrev of %s") % path) as prog:
perftrace.tracevalue("Path", path)
allrevs = repo.revs("_all()")
allrevs.sort(reverse=True)
for i, rev in enumerate(allrevs):
prog.value = i
node = cl.node(rev)
data = cl.read(
node) # get changeset data (we avoid object creation)
if path in data[3]: # checking the 'files' field.
# The file has been touched, check if the hash is what we're
# looking for.
if fileid == mfl[data[0]].read().get(path):
perftrace.tracevalue("Distance", i)
return rev
# Couldn't find the linkrev. This should generally not happen, and will
# likely cause a crash.
return None
def bytesprogresstest(ui):
values = [
0,
10,
250,
999,
1000,
1024,
22000,
1048576,
1474560,
123456789,
555555555,
1000000000,
1111111111,
]
with progress.bar(ui,
"bytes progress test",
"bytes",
max(values),
formatfunc=util.bytecount) as prog:
for value in values:
prog.value = (value, "%s bytes" % value)
progress._engine.pump(_faketime.increment())
def bytesprogresstest(ui):
values = [
0,
10,
250,
999,
1000,
1024,
22000,
1048576,
1474560,
123456789,
555555555,
1000000000,
1111111111,
]
with progress.bar(ui,
"bytes progress test",
"bytes",
max(values),
formatfunc=util.bytecount) as prog:
for value in values:
prog.value = (value, "%s bytes" % value)
syncrender()
def _batch(self, pointers, localstore, action, objectnames=None):
if action not in ["upload", "download"]:
raise error.ProgrammingError("invalid Git-LFS action: %s" % action)
response = self._batchrequest(pointers, action)
objects = self._extractobjects(response, pointers, action)
total = sum(x.get("size", 0) for x in objects)
perftrace.tracebytes("Size", total)
sizes = {}
for obj in objects:
sizes[obj.get("oid")] = obj.get("size", 0)
topic = {
"upload": _("lfs uploading"),
"download": _("lfs downloading")
}[action]
if self.ui.verbose and len(objects) > 1:
self.ui.write(
_("lfs: need to transfer %d objects (%s)\n") %
(len(objects), util.bytecount(total)))
def transfer(chunk):
for obj in chunk:
objsize = obj.get("size", 0)
if self.ui.verbose:
if action == "download":
msg = _("lfs: downloading %s (%s)\n")
elif action == "upload":
msg = _("lfs: uploading %s (%s)\n")
self.ui.write(msg %
(obj.get("oid"), util.bytecount(objsize)))
retry = self.retry
while True:
try:
yield 0, obj.get("oid")
self._basictransfer(obj, action, localstore)
yield 1, obj.get("oid")
break
except Exception as ex:
if retry > 0:
if self.ui.verbose:
self.ui.write(
_("lfs: failed: %r (remaining retry %d)\n")
% (ex, retry))
retry -= 1
continue
raise
starttimestamp = util.timer()
if action == "download":
oids = worker.worker(
self.ui,
0.1,
transfer,
(),
sorted(objects, key=lambda o: o.get("oid")),
preferthreads=True,
callsite="blobstore",
)
else:
oids = transfer(objects)
transferred = 0
with progress.bar(self.ui,
topic,
_("bytes"),
total=total,
formatfunc=util.bytecount) as prog:
for count, oid in oids:
if count != 0:
transferred += sizes[oid]
if self.ui.verbose:
self.ui.write(_("lfs: processed: %s\n") % oid)
if objectnames is not None:
prog.value = (transferred, objectnames.get(oid, ""))
else:
prog.value = transferred
currenttimestamp = util.timer()
self._metrics["lfs_%s_size" % action] += total
self._metrics["lfs_%s_time" % action] += (currenttimestamp - max(
self._timestamp["latest_%s_timestamp" % action],
starttimestamp)) * 1000
self._timestamp["latest_%s_timestamp" % action] = currenttimestamp
def requestpacks(self, fileids, fetchdata, fetchhistory):
self.remotecache.reconnect()
perftrace.traceflag("packs")
cache = self.remotecache
fileslog = self.repo.fileslog
total = len(fileids)
totalfetches = 0
if fetchdata:
totalfetches += total
if fetchhistory:
totalfetches += total
with progress.bar(self.ui,
_("fetching from memcache"),
total=totalfetches) as prog:
# generate `get` keys and make data request
getkeys = [file + "\0" + node for file, node in fileids]
if fetchdata:
cache.getdatapack(getkeys)
if fetchhistory:
cache.gethistorypack(getkeys)
# receive both data and history
misses = []
try:
allmisses = set()
if fetchdata:
allmisses.update(cache.receive(prog))
fileslog.contentstore.markforrefresh()
if fetchhistory:
allmisses.update(cache.receive(prog))
fileslog.metadatastore.markforrefresh()
misses = [key.split("\0") for key in allmisses]
perftrace.tracevalue("Memcache Misses", len(misses))
except CacheConnectionError:
misses = fileids
self.ui.warn(
_("warning: cache connection closed early - " +
"falling back to server\n"))
global fetchmisses
missedfiles = len(misses)
fetchmisses += missedfiles
fromcache = total - missedfiles
self.ui.log(
"remotefilelog",
"remote cache hit rate is %r of %r\n",
fromcache,
total,
hit=fromcache,
total=total,
)
oldumask = os.umask(0o002)
try:
# receive cache misses from master
if missedfiles > 0:
self._fetchpackfiles(misses, fetchdata, fetchhistory)
finally:
os.umask(oldumask)
def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args):
if not requirement in repo.requirements:
return orig(repo, source, revmap, trp, expectedfiles, *args)
newfiles = 0
visited = set()
revisiondatas = {}
queue = []
# Normal Mercurial processes each file one at a time, adding all
# the new revisions for that file at once. In remotefilelog a file
# revision may depend on a different file's revision (in the case
# of a rename/copy), so we must lay all revisions down across all
# files in topological order.
# read all the file chunks but don't add them
with progress.bar(repo.ui, _("files"), total=expectedfiles) as prog:
while True:
chunkdata = source.filelogheader()
if not chunkdata:
break
f = chunkdata["filename"]
repo.ui.debug("adding %s revisions\n" % f)
prog.value += 1
if not repo.shallowmatch(f):
fl = repo.file(f)
deltas = source.deltaiter()
fl.addgroup(deltas, revmap, trp)
continue
chain = None
while True:
# returns: (node, p1, p2, cs, deltabase, delta, flags) or None
revisiondata = source.deltachunk(chain)
if not revisiondata:
break
chain = revisiondata[0]
revisiondatas[(f, chain)] = revisiondata
queue.append((f, chain))
if f not in visited:
newfiles += 1
visited.add(f)
if chain is None:
raise error.Abort(_("received file revlog group is empty"))
processed = set()
def available(f, node, depf, depnode):
if depnode != nullid and (depf, depnode) not in processed:
if not (depf, depnode) in revisiondatas:
# It's not in the changegroup, assume it's already
# in the repo
return True
# re-add self to queue
queue.insert(0, (f, node))
# add dependency in front
queue.insert(0, (depf, depnode))
return False
return True
skipcount = 0
# Prefetch the non-bundled revisions that we will need
prefetchfiles = []
for f, node in queue:
revisiondata = revisiondatas[(f, node)]
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
dependents = [revisiondata[1], revisiondata[2], revisiondata[4]]
for dependent in dependents:
if dependent == nullid or (f, dependent) in revisiondatas:
continue
prefetchfiles.append((f, hex(dependent)))
repo.fileservice.prefetch(prefetchfiles)
# Get rawtext by applying delta chains.
@util.lrucachefunc
def reconstruct(f, node):
revisiondata = revisiondatas.get((f, node), None)
if revisiondata is None:
# Read from repo.
return repo.file(f).revision(node, raw=False)
else:
# Apply delta-chain.
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
deltabase, delta, flags = revisiondata[4:]
if deltabase == nullid:
base = ""
else:
if flags:
# LFS (flags != 0) should always use nullid as deltabase.
raise error.Abort("unexpected deltabase")
base = reconstruct(f, deltabase)
rawtext = mdiff.patch(base, delta)
if isinstance(rawtext, pycompat.buffer): # noqa
rawtext = bytes(rawtext)
return rawtext
# Apply the revisions in topological order such that a revision
# is only written once it's deltabase and parents have been written.
maxskipcount = len(queue) + 1
while queue:
f, node = queue.pop(0)
if (f, node) in processed:
continue
skipcount += 1
if skipcount > maxskipcount:
raise error.Abort(_("circular node dependency on ancestormap"))
revisiondata = revisiondatas[(f, node)]
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
node, p1, p2, linknode, deltabase, delta, flags = revisiondata
# Deltas are always against flags=0 rawtext (see revdiff and its
# callers), if deltabase is not nullid.
if flags and deltabase != nullid:
raise error.Abort("unexpected deltabase")
rawtext = reconstruct(f, node)
meta, text = shallowutil.parsemeta(rawtext, flags)
if "copy" in meta:
copyfrom = meta["copy"]
copynode = bin(meta["copyrev"])
if not available(f, node, copyfrom, copynode):
continue
if any(not available(f, node, f, p) for p in [p1, p2] if p != nullid):
continue
# Use addrawrevision so if it's already LFS, take it as-is, do not
# re-calculate the LFS object.
fl = repo.file(f)
fl.addrawrevision(rawtext,
trp,
linknode,
p1,
p2,
node=node,
flags=flags)
processed.add((f, node))
skipcount = 0
return len(revisiondatas), newfiles
def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
fnodes, source):
# type: (Sequence[int], Mapping[bytes, int], bool, Any, MutableMapping[str, Any], Any) -> Iterable[bytes]
"""
- `commonrevs` is the set of known commits on both sides
- `clrevorder` is a mapping from cl node to rev number, used for
determining which commit is newer.
- `mfs` is the potential manifest nodes to send,
with maps to their linknodes
{ manifest root node -> link node }
- `fnodes` is a mapping of { filepath -> { node -> clnode } }
If fastpathlinkrev is false, we are responsible for populating
fnodes.
- `args` and `kwargs` are extra arguments that will be passed to the
core generatemanifests method, whose length depends on the
version of core Hg.
"""
# If we're not using the fastpath, then all the trees will be necessary
# so we can inspect which files changed and need to be sent. So let's
# bulk fetch the trees up front.
repo = self._repo
if self._cansendflat(mfs.keys()):
# In this code path, generating the manifests populates fnodes for
# us.
chunks = super(shallowcg1packer,
self).generatemanifests(commonrevs, clrevorder,
fastpathlinkrev, mfs,
fnodes, source)
for chunk in chunks:
yield chunk
else:
# If not using the fast path, we need to discover what files to send
if not fastpathlinkrev:
localmfstore = None
if len(repo.manifestlog.localdatastores) > 0:
localmfstore = repo.manifestlog.localdatastores[0]
sharedmfstore = None
if len(repo.manifestlog.shareddatastores) > 0:
sharedmfstore = contentstore.unioncontentstore(
*repo.manifestlog.shareddatastores)
def containslocalfiles(mfnode):
# This is a local tree, then it contains local files.
if localmfstore and not localmfstore.getmissing(
[("", mfnode)]):
return True
# If not a local tree, and it doesn't exist in the store,
# then it is to be generated and may contain local files.
# This can happen while serving an infinitepush bundle that
# contains flat manifests. It will need to generate trees
# for that manifest.
if (repo.svfs.treemanifestserver and sharedmfstore
and sharedmfstore.getmissing([("", mfnode)])):
return True
return False
# If we're sending files, we need to process the manifests
filestosend = self.shouldaddfilegroups(source)
if filestosend is not NoFiles:
mflog = repo.manifestlog
with progress.bar(repo.ui, _("manifests"),
total=len(mfs)) as prog:
for mfnode, clnode in pycompat.iteritems(mfs):
prog.value += 1
if filestosend == LocalFiles and not containslocalfiles(
mfnode):
continue
try:
mfctx = mflog[mfnode]
p1node = mfctx.parents[0]
p1ctx = mflog[p1node]
except LookupError:
if not repo.svfs.treemanifestserver or treeonly(
repo):
raise
# If we can't find the flat version, look for trees
tmfl = mflog.treemanifestlog
mfctx = tmfl[mfnode]
p1node = tmfl[mfnode].parents[0]
p1ctx = tmfl[p1node]
diff = pycompat.iteritems(p1ctx.read().diff(
mfctx.read()))
for filename, ((anode, aflag), (bnode,
bflag)) in diff:
if bnode is not None:
fclnodes = fnodes.setdefault(filename, {})
fclnode = fclnodes.setdefault(
bnode, clnode)
if clrevorder[clnode] < clrevorder[fclnode]:
fclnodes[bnode] = clnode
yield self.close()
def verify(ui, repo, hgctx):
"""verify that a Mercurial rev matches the corresponding Git rev
Given a Mercurial revision that has a corresponding Git revision in the map,
this attempts to answer whether that revision has the same contents as the
corresponding Git revision.
"""
handler = repo.githandler
gitsha = handler.map_git_get(hgctx.hex())
if not gitsha:
# TODO deal better with commits in the middle of octopus merges
raise hgutil.Abort(
_("no git commit found for rev %s") % hgctx,
hint=_("if this is an octopus merge, "
"verify against the last rev"),
)
try:
gitcommit = handler.git.get_object(pycompat.encodeutf8(gitsha))
except KeyError:
raise hgutil.Abort(
_("git equivalent %s for rev %s not found!") % (gitsha, hgctx))
if not isinstance(gitcommit, Commit):
raise hgutil.Abort(
_("git equivalent %s for rev %s is not a commit!") %
(gitsha, hgctx))
ui.status(_("verifying rev %s against git commit %s\n") % (hgctx, gitsha))
failed = False
# TODO check commit message and other metadata
dirkind = stat.S_IFDIR
hgfiles = set(hgctx)
gitfiles = set()
i = 0
with progress.bar(ui, _("verify"), total=len(hgfiles)) as prog:
for gitfile, dummy in diff_tree.walk_trees(handler.git.object_store,
gitcommit.tree, None):
if gitfile.mode == dirkind:
continue
# TODO deal with submodules
if gitfile.mode == S_IFGITLINK:
continue
prog.value = i
i += 1
gitfilepath = pycompat.decodeutf8(gitfile.path)
gitfiles.add(gitfilepath)
try:
fctx = hgctx[gitfilepath]
except error.LookupError:
# we'll deal with this at the end
continue
hgflags = fctx.flags()
gitflags = handler.convert_git_int_mode(gitfile.mode)
if hgflags != gitflags:
ui.write(
_("file has different flags: %s (hg '%s', git '%s')\n") %
(gitfilepath, hgflags, gitflags))
failed = True
if fctx.data() != handler.git[gitfile.sha].data:
ui.write(_("difference in: %s\n") % gitfilepath)
failed = True
if hgfiles != gitfiles:
failed = True
missing = gitfiles - hgfiles
for f in sorted(missing):
ui.write(_("file found in git but not hg: %s\n") % f)
unexpected = hgfiles - gitfiles
for f in sorted(unexpected):
ui.write(_("file found in hg but not git: %s\n") % f)
if failed:
return 1
else:
return 0
def expandpaths(self, rev, paths, parents):
changed, removed = set(), set()
copies = {}
new_module, revnum = revsplit(rev)[1:]
if new_module != self.module:
self.module = new_module
self.reparent(self.module)
with progress.bar(self.ui, _("scanning paths"), _("paths"), len(paths)) as prog:
for i, (path, ent) in enumerate(paths):
prog.value = (i, path)
entrypath = self.getrelpath(path)
kind = self._checkpath(entrypath, revnum)
if kind == svn.core.svn_node_file:
changed.add(self.recode(entrypath))
if not ent.copyfrom_path or not parents:
continue
# Copy sources not in parent revisions cannot be
# represented, ignore their origin for now
pmodule, prevnum = revsplit(parents[0])[1:]
if ent.copyfrom_rev < prevnum:
continue
copyfrom_path = self.getrelpath(ent.copyfrom_path, pmodule)
if not copyfrom_path:
continue
self.ui.debug(
"copied to %s from %s@%s\n"
% (entrypath, copyfrom_path, ent.copyfrom_rev)
)
copies[self.recode(entrypath)] = self.recode(copyfrom_path)
elif kind == 0: # gone, but had better be a deleted *file*
self.ui.debug("gone from %s\n" % ent.copyfrom_rev)
pmodule, prevnum = revsplit(parents[0])[1:]
parentpath = pmodule + "/" + entrypath
fromkind = self._checkpath(entrypath, prevnum, pmodule)
if fromkind == svn.core.svn_node_file:
removed.add(self.recode(entrypath))
elif fromkind == svn.core.svn_node_dir:
oroot = parentpath.strip("/")
nroot = path.strip("/")
children = self._iterfiles(oroot, prevnum)
for childpath in children:
childpath = childpath.replace(oroot, nroot)
childpath = self.getrelpath("/" + childpath, pmodule)
if childpath:
removed.add(self.recode(childpath))
else:
self.ui.debug(
"unknown path in revision %d: %s\n" % (revnum, path)
)
elif kind == svn.core.svn_node_dir:
if ent.action == "M":
# If the directory just had a prop change,
# then we shouldn't need to look for its children.
continue
if ent.action == "R" and parents:
# If a directory is replacing a file, mark the previous
# file as deleted
pmodule, prevnum = revsplit(parents[0])[1:]
pkind = self._checkpath(entrypath, prevnum, pmodule)
if pkind == svn.core.svn_node_file:
removed.add(self.recode(entrypath))
elif pkind == svn.core.svn_node_dir:
# We do not know what files were kept or removed,
# mark them all as changed.
for childpath in self._iterfiles(pmodule, prevnum):
childpath = self.getrelpath("/" + childpath)
if childpath:
changed.add(self.recode(childpath))
for childpath in self._iterfiles(path, revnum):
childpath = self.getrelpath("/" + childpath)
if childpath:
changed.add(self.recode(childpath))
# Handle directory copies
if not ent.copyfrom_path or not parents:
continue
# Copy sources not in parent revisions cannot be
# represented, ignore their origin for now
pmodule, prevnum = revsplit(parents[0])[1:]
if ent.copyfrom_rev < prevnum:
continue
copyfrompath = self.getrelpath(ent.copyfrom_path, pmodule)
if not copyfrompath:
continue
self.ui.debug(
"mark %s came from %s:%d\n"
% (path, copyfrompath, ent.copyfrom_rev)
)
children = self._iterfiles(ent.copyfrom_path, ent.copyfrom_rev)
for childpath in children:
childpath = self.getrelpath("/" + childpath, pmodule)
if not childpath:
continue
copytopath = path + childpath[len(copyfrompath) :]
copytopath = self.getrelpath(copytopath)
copies[self.recode(copytopath)] = self.recode(childpath)
changed.update(removed)
return (list(changed), removed, copies)
def _prefetch(self,
revs,
base=None,
pats=None,
opts=None,
matcher=None):
fallbackpath = self.fallbackpath
if fallbackpath:
# If we know a rev is on the server, we should fetch the server
# version of those files, since our local file versions might
# become obsolete if the local commits are stripped.
with progress.spinner(self.ui,
_("finding outgoing revisions")):
localrevs = self.revs("outgoing(%s)", fallbackpath)
if base is not None and base != nullrev:
serverbase = list(
self.revs("first(reverse(::%s) - %ld)", base,
localrevs))
if serverbase:
base = serverbase[0]
else:
localrevs = self
mfl = self.manifestlog
if base is not None:
mfdict = mfl[self[base].manifestnode()].read()
skip = set(mfdict.iteritems())
else:
skip = set()
# Copy the skip set to start large and avoid constant resizing,
# and since it's likely to be very similar to the prefetch set.
files = skip.copy()
serverfiles = skip.copy()
visited = set()
visited.add(nullid)
with progress.bar(self.ui, _("prefetching"),
total=len(revs)) as prog:
for rev in sorted(revs):
ctx = self[rev]
if pats:
m = scmutil.match(ctx, pats, opts)
if matcher is None:
matcher = self.maybesparsematch(rev)
mfnode = ctx.manifestnode()
mfctx = mfl[mfnode]
# Decompressing manifests is expensive.
# When possible, only read the deltas.
p1, p2 = mfctx.parents
if p1 in visited and p2 in visited:
mfdict = mfctx.readnew()
else:
mfdict = mfctx.read()
diff = mfdict.iteritems()
if pats:
diff = (pf for pf in diff if m(pf[0]))
if matcher:
diff = (pf for pf in diff if matcher(pf[0]))
if rev not in localrevs:
serverfiles.update(diff)
else:
files.update(diff)
visited.add(mfctx.node())
prog.value += 1
files.difference_update(skip)
serverfiles.difference_update(skip)
# Fetch files known to be on the server
if serverfiles:
results = [(path, hex(fnode)) for (path, fnode) in serverfiles]
self.fileservice.prefetch(results, force=True)
# Fetch files that may or may not be on the server
if files:
results = [(path, hex(fnode)) for (path, fnode) in files]
self.fileservice.prefetch(results)
