def setupctx(ctx):
"""return a 'makectx' function suitable for checkcopies usage from ctx
We have to re-setup the function building 'filectx' for each
'checkcopies' to ensure the linkrev adjustement is properly setup for
each. Linkrev adjustment is important to avoid bug in rename
detection. Moreover, having a proper '_ancestrycontext' setup ensures
the performance impact of this adjustment is kept limited. Without it,
each file could do a full dag traversal making the time complexity of
the operation explode (see issue4537).
This function exists here mostly to limit the impact on stable. Feel
free to refactor on default.
"""
rev = ctx.rev()
ac = getattr(ctx, '_ancestrycontext', None)
if ac is None:
revs = [rev]
if rev is None:
revs = [p.rev() for p in ctx.parents()]
ac = ctx._repo.changelog.ancestors(revs, inclusive=True)
ctx._ancestrycontext = ac
def makectx(f, n):
if len(n) != 20: # in a working context?
if c1.rev() is None:
return c1.filectx(f)
return c2.filectx(f)
fctx = repo.filectx(f, fileid=n)
# setup only needed for filectx not create from a changectx
fctx._ancestrycontext = ac
fctx._descendantrev = rev
return fctx
return util.lrucachefunc(makectx)
def _verify(repo):
repo = repo.unfiltered()
mflinkrevs = {}
filelinkrevs = {}
filenodes = {}
revisions = 0
badrevs = set()
errors = [0]
warnings = [0]
ui = repo.ui
cl = repo.changelog
mf = repo.manifest
lrugetctx = util.lrucachefunc(repo.changectx)
if not repo.url().startswith('file:'):
raise util.Abort(_("cannot verify bundle or remote repos"))
def err(linkrev, msg, filename=None):
if linkrev is not None:
badrevs.add(linkrev)
else:
linkrev = '?'
msg = "%s: %s" % (linkrev, msg)
if filename:
msg = "%s@%s" % (filename, msg)
ui.warn(" " + msg + "\n")
errors[0] += 1
def exc(linkrev, msg, inst, filename=None):
if isinstance(inst, KeyboardInterrupt):
ui.warn(_("interrupted"))
raise
if not str(inst):
inst = repr(inst)
err(linkrev, "%s: %s" % (msg, inst), filename)
def warn(msg):
ui.warn(msg + "\n")
warnings[0] += 1
def checklog(obj, name, linkrev):
if not len(obj) and (havecl or havemf):
err(linkrev, _("empty or missing %s") % name)
return
d = obj.checksize()
if d[0]:
err(None, _("data length off by %d bytes") % d[0], name)
if d[1]:
err(None, _("index contains %d extra bytes") % d[1], name)
if obj.version != revlog.REVLOGV0:
if not revlogv1:
warn(_("warning: `%s' uses revlog format 1") % name)
elif revlogv1:
warn(_("warning: `%s' uses revlog format 0") % name)
def checkentry(obj, i, node, seen, linkrevs, f):
lr = obj.linkrev(obj.rev(node))
if lr < 0 or (havecl and lr not in linkrevs):
if lr < 0 or lr >= len(cl):
msg = _("rev %d points to nonexistent changeset %d")
else:
msg = _("rev %d points to unexpected changeset %d")
err(None, msg % (i, lr), f)
if linkrevs:
if f and len(linkrevs) > 1:
try:
# attempt to filter down to real linkrevs
linkrevs = [
l for l in linkrevs
if lrugetctx(l)[f].filenode() == node
]
except Exception:
pass
warn(_(" (expected %s)") % " ".join(map(str, linkrevs)))
lr = None # can't be trusted
try:
p1, p2 = obj.parents(node)
if p1 not in seen and p1 != nullid:
err(lr,
_("unknown parent 1 %s of %s") % (short(p1), short(node)),
f)
if p2 not in seen and p2 != nullid:
err(lr,
_("unknown parent 2 %s of %s") % (short(p2), short(node)),
f)
except Exception as inst:
exc(lr, _("checking parents of %s") % short(node), inst, f)
if node in seen:
err(lr, _("duplicate revision %d (%d)") % (i, seen[node]), f)
seen[node] = i
return lr
if os.path.exists(repo.sjoin("journal")):
ui.warn(_("abandoned transaction found - run hg recover\n"))
revlogv1 = cl.version != revlog.REVLOGV0
if ui.verbose or not revlogv1:
ui.status(
_("repository uses revlog format %d\n") % (revlogv1 and 1 or 0))
havecl = len(cl) > 0
havemf = len(mf) > 0
ui.status(_("checking changesets\n"))
refersmf = False
seen = {}
checklog(cl, "changelog", 0)
total = len(repo)
for i in repo:
ui.progress(_('checking'), i, total=total, unit=_('changesets'))
n = cl.node(i)
checkentry(cl, i, n, seen, [i], "changelog")
try:
changes = cl.read(n)
if changes[0] != nullid:
mflinkrevs.setdefault(changes[0], []).append(i)
refersmf = True
for f in changes[3]:
filelinkrevs.setdefault(_normpath(f), []).append(i)
except Exception as inst:
refersmf = True
exc(i, _("unpacking changeset %s") % short(n), inst)
ui.progress(_('checking'), None)
ui.status(_("checking manifests\n"))
seen = {}
if refersmf:
# Do not check manifest if there are only changelog entries with
# null manifests.
checklog(mf, "manifest", 0)
total = len(mf)
for i in mf:
ui.progress(_('checking'), i, total=total, unit=_('manifests'))
n = mf.node(i)
lr = checkentry(mf, i, n, seen, mflinkrevs.get(n, []), "manifest")
if n in mflinkrevs:
del mflinkrevs[n]
else:
err(lr, _("%s not in changesets") % short(n), "manifest")
try:
for f, fn in mf.readdelta(n).iteritems():
if not f:
err(lr, _("file without name in manifest"))
elif f != "/dev/null": # ignore this in very old repos
filenodes.setdefault(_normpath(f), {}).setdefault(fn, lr)
except Exception as inst:
exc(lr, _("reading manifest delta %s") % short(n), inst)
ui.progress(_('checking'), None)
ui.status(_("crosschecking files in changesets and manifests\n"))
total = len(mflinkrevs) + len(filelinkrevs) + len(filenodes)
count = 0
if havemf:
for c, m in sorted([(c, m) for m in mflinkrevs
for c in mflinkrevs[m]]):
count += 1
if m == nullid:
continue
ui.progress(_('crosschecking'), count, total=total)
err(c, _("changeset refers to unknown manifest %s") % short(m))
mflinkrevs = None # del is bad here due to scope issues
for f in sorted(filelinkrevs):
count += 1
ui.progress(_('crosschecking'), count, total=total)
if f not in filenodes:
lr = filelinkrevs[f][0]
err(lr, _("in changeset but not in manifest"), f)
if havecl:
for f in sorted(filenodes):
count += 1
ui.progress(_('crosschecking'), count, total=total)
if f not in filelinkrevs:
try:
fl = repo.file(f)
lr = min([fl.linkrev(fl.rev(n)) for n in filenodes[f]])
except Exception:
lr = None
err(lr, _("in manifest but not in changeset"), f)
ui.progress(_('crosschecking'), None)
ui.status(_("checking files\n"))
storefiles = set()
for f, f2, size in repo.store.datafiles():
if not f:
err(None, _("cannot decode filename '%s'") % f2)
elif size > 0 or not revlogv1:
storefiles.add(_normpath(f))
fncachewarned = False
files = sorted(set(filenodes) | set(filelinkrevs))
total = len(files)
for i, f in enumerate(files):
ui.progress(_('checking'), i, item=f, total=total)
try:
linkrevs = filelinkrevs[f]
except KeyError:
# in manifest but not in changelog
linkrevs = []
if linkrevs:
lr = linkrevs[0]
else:
lr = None
try:
fl = repo.file(f)
except error.RevlogError as e:
err(lr, _("broken revlog! (%s)") % e, f)
continue
for ff in fl.files():
try:
storefiles.remove(ff)
except KeyError:
warn(_(" warning: revlog '%s' not in fncache!") % ff)
fncachewarned = True
checklog(fl, f, lr)
seen = {}
rp = None
for i in fl:
revisions += 1
n = fl.node(i)
lr = checkentry(fl, i, n, seen, linkrevs, f)
if f in filenodes:
if havemf and n not in filenodes[f]:
err(lr, _("%s not in manifests") % (short(n)), f)
else:
del filenodes[f][n]
# verify contents
try:
l = len(fl.read(n))
rp = fl.renamed(n)
if l != fl.size(i):
if len(fl.revision(n)) != fl.size(i):
err(
lr,
_("unpacked size is %s, %s expected") %
(l, fl.size(i)), f)
except error.CensoredNodeError:
if ui.config("censor", "policy", "abort") == "abort":
err(lr, _("censored file data"), f)
except Exception as inst:
exc(lr, _("unpacking %s") % short(n), inst, f)
# check renames
try:
if rp:
if lr is not None and ui.verbose:
ctx = lrugetctx(lr)
found = False
for pctx in ctx.parents():
if rp[0] in pctx:
found = True
break
if not found:
warn(
_("warning: copy source of '%s' not"
" in parents of %s") % (f, ctx))
fl2 = repo.file(rp[0])
if not len(fl2):
err(
lr,
_("empty or missing copy source revlog %s:%s") %
(rp[0], short(rp[1])), f)
elif rp[1] == nullid:
ui.note(
_("warning: %s@%s: copy source"
" revision is nullid %s:%s\n") %
(f, lr, rp[0], short(rp[1])))
else:
fl2.rev(rp[1])
except Exception as inst:
exc(lr, _("checking rename of %s") % short(n), inst, f)
# cross-check
if f in filenodes:
fns = [(lr, n) for n, lr in filenodes[f].iteritems()]
for lr, node in sorted(fns):
err(lr, _("%s in manifests not found") % short(node), f)
ui.progress(_('checking'), None)
for f in storefiles:
warn(_("warning: orphan revlog '%s'") % f)
ui.status(
_("%d files, %d changesets, %d total revisions\n") %
(len(files), len(cl), revisions))
if warnings[0]:
ui.warn(_("%d warnings encountered!\n") % warnings[0])
if fncachewarned:
ui.warn(
_('hint: run "hg debugrebuildfncache" to recover from '
'corrupt fncache\n'))
if errors[0]:
ui.warn(_("%d integrity errors encountered!\n") % errors[0])
if badrevs:
ui.warn(
_("(first damaged changeset appears to be %d)\n") %
min(badrevs))
return 1
def annotate(self, follow=False, linenumber=None, diffopts=None):
'''returns a list of tuples of (ctx, line) for each line
in the file, where ctx is the filectx of the node where
that line was last changed.
This returns tuples of ((ctx, linenumber), line) for each line,
if "linenumber" parameter is NOT "None".
In such tuples, linenumber means one at the first appearance
in the managed file.
To reduce annotation cost,
this returns fixed value(False is used) as linenumber,
if "linenumber" parameter is "False".'''
def decorate_compat(text, rev):
return ([rev] * len(text.splitlines()), text)
def without_linenumber(text, rev):
return ([(rev, False)] * len(text.splitlines()), text)
def with_linenumber(text, rev):
size = len(text.splitlines())
return ([(rev, i) for i in xrange(1, size + 1)], text)
decorate = (((linenumber is None) and decorate_compat)
or (linenumber and with_linenumber) or without_linenumber)
def pair(parent, child):
blocks = mdiff.allblocks(parent[1],
child[1],
opts=diffopts,
refine=True)
for (a1, a2, b1, b2), t in blocks:
# Changed blocks ('!') or blocks made only of blank lines ('~')
# belong to the child.
if t == '=':
child[0][b1:b2] = parent[0][a1:a2]
return child
getlog = util.lrucachefunc(lambda x: self._repo.file(x))
def getctx(path, fileid):
log = path == self._path and self._filelog or getlog(path)
return filectx(self._repo, path, fileid=fileid, filelog=log)
getctx = util.lrucachefunc(getctx)
def parents(f):
# we want to reuse filectx objects as much as possible
p = f._path
if f._filerev is None: # working dir
pl = [(n.path(), n.filerev()) for n in f.parents()]
else:
pl = [(p, n) for n in f._filelog.parentrevs(f._filerev)]
if follow:
r = f.renamed()
if r:
pl[0] = (r[0], getlog(r[0]).rev(r[1]))
return [getctx(p, n) for p, n in pl if n != nullrev]
# use linkrev to find the first changeset where self appeared
if self.rev() != self.linkrev():
base = self.filectx(self.filerev())
else:
base = self
# This algorithm would prefer to be recursive, but Python is a
# bit recursion-hostile. Instead we do an iterative
# depth-first search.
visit = [base]
hist = {}
pcache = {}
needed = {base: 1}
while visit:
f = visit[-1]
if f not in pcache:
pcache[f] = parents(f)
ready = True
pl = pcache[f]
for p in pl:
if p not in hist:
ready = False
visit.append(p)
needed[p] = needed.get(p, 0) + 1
if ready:
visit.pop()
curr = decorate(f.data(), f)
for p in pl:
curr = pair(hist[p], curr)
if needed[p] == 1:
del hist[p]
else:
needed[p] -= 1
hist[f] = curr
pcache[f] = []
return zip(hist[base][0], hist[base][1].splitlines(True))
def copies(repo, c1, c2, ca, checkdirs=False):
"""
Find moves and copies between context c1 and c2
"""
# avoid silly behavior for update from empty dir
if not c1 or not c2 or c1 == c2:
return {}, {}
# avoid silly behavior for parent -> working dir
if c2.node() is None and c1.node() == repo.dirstate.parents()[0]:
return repo.dirstate.copies(), {}
limit = _findlimit(repo, c1.rev(), c2.rev())
m1 = c1.manifest()
m2 = c2.manifest()
ma = ca.manifest()
def makectx(f, n):
if len(n) != 20: # in a working context?
if c1.rev() is None:
return c1.filectx(f)
return c2.filectx(f)
return repo.filectx(f, fileid=n)
ctx = util.lrucachefunc(makectx)
copy = {}
fullcopy = {}
diverge = {}
def checkcopies(f, m1, m2):
'''check possible copies of f from m1 to m2'''
c1 = ctx(f, m1[f])
for of in _findoldnames(c1, limit):
fullcopy[f] = of # remember for dir rename detection
if of in m2: # original file not in other manifest?
# if the original file is unchanged on the other branch,
# no merge needed
if m2[of] != ma.get(of):
c2 = ctx(of, m2[of])
ca = c1.ancestor(c2)
# related and named changed on only one side?
if ca and (ca.path() == f or ca.path() == c2.path()):
if c1 != ca or c2 != ca: # merge needed?
copy[f] = of
elif of in ma:
diverge.setdefault(of, []).append(f)
repo.ui.debug(_(" searching for copies back to rev %d\n") % limit)
u1 = _nonoverlap(m1, m2, ma)
u2 = _nonoverlap(m2, m1, ma)
if u1:
repo.ui.debug(
_(" unmatched files in local:\n %s\n") % "\n ".join(u1))
if u2:
repo.ui.debug(
_(" unmatched files in other:\n %s\n") % "\n ".join(u2))
for f in u1:
checkcopies(f, m1, m2)
for f in u2:
checkcopies(f, m2, m1)
diverge2 = set()
for of, fl in diverge.items():
if len(fl) == 1:
del diverge[of] # not actually divergent
else:
diverge2.update(fl) # reverse map for below
if fullcopy:
repo.ui.debug(_(" all copies found (* = to merge, ! = divergent):\n"))
for f in fullcopy:
note = ""
if f in copy: note += "*"
if f in diverge2: note += "!"
repo.ui.debug(" %s -> %s %s\n" % (f, fullcopy[f], note))
del diverge2
if not fullcopy or not checkdirs:
return copy, diverge
repo.ui.debug(_(" checking for directory renames\n"))
# generate a directory move map
d1, d2 = _dirs(m1), _dirs(m2)
invalid = set()
dirmove = {}
# examine each file copy for a potential directory move, which is
# when all the files in a directory are moved to a new directory
for dst, src in fullcopy.iteritems():
dsrc, ddst = _dirname(src), _dirname(dst)
if dsrc in invalid:
# already seen to be uninteresting
continue
elif dsrc in d1 and ddst in d1:
# directory wasn't entirely moved locally
invalid.add(dsrc)
elif dsrc in d2 and ddst in d2:
# directory wasn't entirely moved remotely
invalid.add(dsrc)
elif dsrc in dirmove and dirmove[dsrc] != ddst:
# files from the same directory moved to two different places
invalid.add(dsrc)
else:
# looks good so far
dirmove[dsrc + "/"] = ddst + "/"
for i in invalid:
if i in dirmove:
del dirmove[i]
del d1, d2, invalid
if not dirmove:
return copy, diverge
for d in dirmove:
repo.ui.debug(_(" dir %s -> %s\n") % (d, dirmove[d]))
# check unaccounted nonoverlapping files against directory moves
for f in u1 + u2:
if f not in fullcopy:
for d in dirmove:
if f.startswith(d):
# new file added in a directory that was moved, move it
df = dirmove[d] + f[len(d):]
if df not in copy:
copy[f] = df
repo.ui.debug(_(" file %s -> %s\n") % (f, copy[f]))
break
return copy, diverge
def mergecopies(repo, c1, c2, ca):
"""
Find moves and copies between context c1 and c2 that are relevant
for merging.
Returns four dicts: "copy", "movewithdir", "diverge", and
"renamedelete".
"copy" is a mapping from destination name -> source name,
where source is in c1 and destination is in c2 or vice-versa.
"movewithdir" is a mapping from source name -> destination name,
where the file at source present in one context but not the other
needs to be moved to destination by the merge process, because the
other context moved the directory it is in.
"diverge" is a mapping of source name -> list of destination names
for divergent renames.
"renamedelete" is a mapping of source name -> list of destination
names for files deleted in c1 that were renamed in c2 or vice-versa.
"""
# avoid silly behavior for update from empty dir
if not c1 or not c2 or c1 == c2:
return {}, {}, {}, {}
# avoid silly behavior for parent -> working dir
if c2.node() is None and c1.node() == repo.dirstate.p1():
return repo.dirstate.copies(), {}, {}, {}
limit = _findlimit(repo, c1.rev(), c2.rev())
if limit is None:
# no common ancestor, no copies
return {}, {}, {}, {}
m1 = c1.manifest()
m2 = c2.manifest()
ma = ca.manifest()
def makectx(f, n):
if len(n) != 20: # in a working context?
if c1.rev() is None:
return c1.filectx(f)
return c2.filectx(f)
return repo.filectx(f, fileid=n)
ctx = util.lrucachefunc(makectx)
copy = {}
movewithdir = {}
fullcopy = {}
diverge = {}
def _checkcopies(f, m1, m2):
checkcopies(ctx, f, m1, m2, ca, limit, diverge, copy, fullcopy)
repo.ui.debug(" searching for copies back to rev %d\n" % limit)
u1 = _nonoverlap(m1, m2, ma)
u2 = _nonoverlap(m2, m1, ma)
if u1:
repo.ui.debug(" unmatched files in local:\n %s\n"
% "\n ".join(u1))
if u2:
repo.ui.debug(" unmatched files in other:\n %s\n"
% "\n ".join(u2))
for f in u1:
_checkcopies(f, m1, m2)
for f in u2:
_checkcopies(f, m2, m1)
renamedelete = {}
renamedelete2 = set()
diverge2 = set()
for of, fl in diverge.items():
if len(fl) == 1 or of in c1 or of in c2:
del diverge[of] # not actually divergent, or not a rename
if of not in c1 and of not in c2:
# renamed on one side, deleted on the other side, but filter
# out files that have been renamed and then deleted
renamedelete[of] = [f for f in fl if f in c1 or f in c2]
renamedelete2.update(fl) # reverse map for below
else:
diverge2.update(fl) # reverse map for below
if fullcopy:
repo.ui.debug(" all copies found (* = to merge, ! = divergent, "
"% = renamed and deleted):\n")
for f in sorted(fullcopy):
note = ""
if f in copy:
note += "*"
if f in diverge2:
note += "!"
if f in renamedelete2:
note += "%"
repo.ui.debug(" src: '%s' -> dst: '%s' %s\n" % (fullcopy[f], f,
note))
del diverge2
if not fullcopy:
return copy, movewithdir, diverge, renamedelete
repo.ui.debug(" checking for directory renames\n")
# generate a directory move map
d1, d2 = c1.dirs(), c2.dirs()
d1.addpath('/')
d2.addpath('/')
invalid = set()
dirmove = {}
# examine each file copy for a potential directory move, which is
# when all the files in a directory are moved to a new directory
for dst, src in fullcopy.iteritems():
dsrc, ddst = _dirname(src), _dirname(dst)
if dsrc in invalid:
# already seen to be uninteresting
continue
elif dsrc in d1 and ddst in d1:
# directory wasn't entirely moved locally
invalid.add(dsrc)
elif dsrc in d2 and ddst in d2:
# directory wasn't entirely moved remotely
invalid.add(dsrc)
elif dsrc in dirmove and dirmove[dsrc] != ddst:
# files from the same directory moved to two different places
invalid.add(dsrc)
else:
# looks good so far
dirmove[dsrc + "/"] = ddst + "/"
for i in invalid:
if i in dirmove:
del dirmove[i]
del d1, d2, invalid
if not dirmove:
return copy, movewithdir, diverge, renamedelete
for d in dirmove:
repo.ui.debug(" discovered dir src: '%s' -> dst: '%s'\n" %
(d, dirmove[d]))
# check unaccounted nonoverlapping files against directory moves
for f in u1 + u2:
if f not in fullcopy:
for d in dirmove:
if f.startswith(d):
# new file added in a directory that was moved, move it
df = dirmove[d] + f[len(d):]
if df not in copy:
movewithdir[f] = df
repo.ui.debug((" pending file src: '%s' -> "
"dst: '%s'\n") % (f, df))
break
return copy, movewithdir, diverge, renamedelete
def _verify(repo):
mflinkrevs = {}
filelinkrevs = {}
filenodes = {}
revisions = 0
badrevs = set()
errors = [0]
warnings = [0]
ui = repo.ui
cl = repo.changelog
mf = repo.manifest
lrugetctx = util.lrucachefunc(repo.changectx)
if not repo.cancopy():
raise util.Abort(_("cannot verify bundle or remote repos"))
def err(linkrev, msg, filename=None):
if linkrev != None:
badrevs.add(linkrev)
else:
linkrev = '?'
msg = "%s: %s" % (linkrev, msg)
if filename:
msg = "%s@%s" % (filename, msg)
ui.warn(" " + msg + "\n")
errors[0] += 1
def exc(linkrev, msg, inst, filename=None):
if isinstance(inst, KeyboardInterrupt):
ui.warn(_("interrupted"))
raise
if not str(inst):
inst = repr(inst)
err(linkrev, "%s: %s" % (msg, inst), filename)
def warn(msg):
ui.warn(msg + "\n")
warnings[0] += 1
def checklog(obj, name, linkrev):
if not len(obj) and (havecl or havemf):
err(linkrev, _("empty or missing %s") % name)
return
d = obj.checksize()
if d[0]:
err(None, _("data length off by %d bytes") % d[0], name)
if d[1]:
err(None, _("index contains %d extra bytes") % d[1], name)
if obj.version != revlog.REVLOGV0:
if not revlogv1:
warn(_("warning: `%s' uses revlog format 1") % name)
elif revlogv1:
warn(_("warning: `%s' uses revlog format 0") % name)
def checkentry(obj, i, node, seen, linkrevs, f):
lr = obj.linkrev(obj.rev(node))
if lr < 0 or (havecl and lr not in linkrevs):
if lr < 0 or lr >= len(cl):
msg = _("rev %d points to nonexistent changeset %d")
else:
msg = _("rev %d points to unexpected changeset %d")
err(None, msg % (i, lr), f)
if linkrevs:
if f and len(linkrevs) > 1:
try:
# attempt to filter down to real linkrevs
linkrevs = [l for l in linkrevs
if lrugetctx(l)[f].filenode() == node]
except:
pass
warn(_(" (expected %s)") % " ".join(map(str, linkrevs)))
lr = None # can't be trusted
try:
p1, p2 = obj.parents(node)
if p1 not in seen and p1 != nullid:
err(lr, _("unknown parent 1 %s of %s") %
(short(p1), short(n)), f)
if p2 not in seen and p2 != nullid:
err(lr, _("unknown parent 2 %s of %s") %
(short(p2), short(p1)), f)
except Exception, inst:
exc(lr, _("checking parents of %s") % short(node), inst, f)
if node in seen:
err(lr, _("duplicate revision %d (%d)") % (i, seen[n]), f)
seen[n] = i
return lr
def _verify(repo):
repo = repo.unfiltered()
mflinkrevs = {}
filelinkrevs = {}
filenodes = {}
revisions = 0
badrevs = set()
errors = [0]
warnings = [0]
ui = repo.ui
cl = repo.changelog
mf = repo.manifest
lrugetctx = util.lrucachefunc(repo.changectx)
if not repo.url().startswith('file:'):
raise util.Abort(_("cannot verify bundle or remote repos"))
def err(linkrev, msg, filename=None):
if linkrev is not None:
badrevs.add(linkrev)
else:
linkrev = '?'
msg = "%s: %s" % (linkrev, msg)
if filename:
msg = "%s@%s" % (filename, msg)
ui.warn(" " + msg + "\n")
errors[0] += 1
def exc(linkrev, msg, inst, filename=None):
if isinstance(inst, KeyboardInterrupt):
ui.warn(_("interrupted"))
raise
if not str(inst):
inst = repr(inst)
err(linkrev, "%s: %s" % (msg, inst), filename)
def warn(msg):
ui.warn(msg + "\n")
warnings[0] += 1
def checklog(obj, name, linkrev):
if not len(obj) and (havecl or havemf):
err(linkrev, _("empty or missing %s") % name)
return
d = obj.checksize()
if d[0]:
err(None, _("data length off by %d bytes") % d[0], name)
if d[1]:
err(None, _("index contains %d extra bytes") % d[1], name)
if obj.version != revlog.REVLOGV0:
if not revlogv1:
warn(_("warning: `%s' uses revlog format 1") % name)
elif revlogv1:
warn(_("warning: `%s' uses revlog format 0") % name)
def checkentry(obj, i, node, seen, linkrevs, f):
lr = obj.linkrev(obj.rev(node))
if lr < 0 or (havecl and lr not in linkrevs):
if lr < 0 or lr >= len(cl):
msg = _("rev %d points to nonexistent changeset %d")
else:
msg = _("rev %d points to unexpected changeset %d")
err(None, msg % (i, lr), f)
if linkrevs:
if f and len(linkrevs) > 1:
try:
# attempt to filter down to real linkrevs
linkrevs = [l for l in linkrevs
if lrugetctx(l)[f].filenode() == node]
except Exception:
pass
warn(_(" (expected %s)") % " ".join(map(str, linkrevs)))
lr = None # can't be trusted
try:
p1, p2 = obj.parents(node)
if p1 not in seen and p1 != nullid:
err(lr, _("unknown parent 1 %s of %s") %
(short(p1), short(node)), f)
if p2 not in seen and p2 != nullid:
err(lr, _("unknown parent 2 %s of %s") %
(short(p2), short(node)), f)
except Exception as inst:
exc(lr, _("checking parents of %s") % short(node), inst, f)
if node in seen:
err(lr, _("duplicate revision %d (%d)") % (i, seen[node]), f)
seen[node] = i
return lr
if os.path.exists(repo.sjoin("journal")):
ui.warn(_("abandoned transaction found - run hg recover\n"))
revlogv1 = cl.version != revlog.REVLOGV0
if ui.verbose or not revlogv1:
ui.status(_("repository uses revlog format %d\n") %
(revlogv1 and 1 or 0))
havecl = len(cl) > 0
havemf = len(mf) > 0
ui.status(_("checking changesets\n"))
refersmf = False
seen = {}
checklog(cl, "changelog", 0)
total = len(repo)
for i in repo:
ui.progress(_('checking'), i, total=total, unit=_('changesets'))
n = cl.node(i)
checkentry(cl, i, n, seen, [i], "changelog")
try:
changes = cl.read(n)
if changes[0] != nullid:
mflinkrevs.setdefault(changes[0], []).append(i)
refersmf = True
for f in changes[3]:
filelinkrevs.setdefault(_normpath(f), []).append(i)
except Exception as inst:
refersmf = True
exc(i, _("unpacking changeset %s") % short(n), inst)
ui.progress(_('checking'), None)
ui.status(_("checking manifests\n"))
seen = {}
if refersmf:
# Do not check manifest if there are only changelog entries with
# null manifests.
checklog(mf, "manifest", 0)
total = len(mf)
for i in mf:
ui.progress(_('checking'), i, total=total, unit=_('manifests'))
n = mf.node(i)
lr = checkentry(mf, i, n, seen, mflinkrevs.get(n, []), "manifest")
if n in mflinkrevs:
del mflinkrevs[n]
else:
err(lr, _("%s not in changesets") % short(n), "manifest")
try:
for f, fn in mf.readdelta(n).iteritems():
if not f:
err(lr, _("file without name in manifest"))
elif f != "/dev/null": # ignore this in very old repos
filenodes.setdefault(_normpath(f), {}).setdefault(fn, lr)
except Exception as inst:
exc(lr, _("reading manifest delta %s") % short(n), inst)
ui.progress(_('checking'), None)
ui.status(_("crosschecking files in changesets and manifests\n"))
total = len(mflinkrevs) + len(filelinkrevs) + len(filenodes)
count = 0
if havemf:
for c, m in sorted([(c, m) for m in mflinkrevs
for c in mflinkrevs[m]]):
count += 1
if m == nullid:
continue
ui.progress(_('crosschecking'), count, total=total)
err(c, _("changeset refers to unknown manifest %s") % short(m))
mflinkrevs = None # del is bad here due to scope issues
for f in sorted(filelinkrevs):
count += 1
ui.progress(_('crosschecking'), count, total=total)
if f not in filenodes:
lr = filelinkrevs[f][0]
err(lr, _("in changeset but not in manifest"), f)
if havecl:
for f in sorted(filenodes):
count += 1
ui.progress(_('crosschecking'), count, total=total)
if f not in filelinkrevs:
try:
fl = repo.file(f)
lr = min([fl.linkrev(fl.rev(n)) for n in filenodes[f]])
except Exception:
lr = None
err(lr, _("in manifest but not in changeset"), f)
ui.progress(_('crosschecking'), None)
ui.status(_("checking files\n"))
storefiles = set()
for f, f2, size in repo.store.datafiles():
if not f:
err(None, _("cannot decode filename '%s'") % f2)
elif size > 0 or not revlogv1:
storefiles.add(_normpath(f))
fncachewarned = False
files = sorted(set(filenodes) | set(filelinkrevs))
total = len(files)
for i, f in enumerate(files):
ui.progress(_('checking'), i, item=f, total=total)
try:
linkrevs = filelinkrevs[f]
except KeyError:
# in manifest but not in changelog
linkrevs = []
if linkrevs:
lr = linkrevs[0]
else:
lr = None
try:
fl = repo.file(f)
except error.RevlogError as e:
err(lr, _("broken revlog! (%s)") % e, f)
continue
for ff in fl.files():
try:
storefiles.remove(ff)
except KeyError:
warn(_(" warning: revlog '%s' not in fncache!") % ff)
fncachewarned = True
checklog(fl, f, lr)
seen = {}
rp = None
for i in fl:
revisions += 1
n = fl.node(i)
lr = checkentry(fl, i, n, seen, linkrevs, f)
if f in filenodes:
if havemf and n not in filenodes[f]:
err(lr, _("%s not in manifests") % (short(n)), f)
else:
del filenodes[f][n]
# verify contents
try:
l = len(fl.read(n))
rp = fl.renamed(n)
if l != fl.size(i):
if len(fl.revision(n)) != fl.size(i):
err(lr, _("unpacked size is %s, %s expected") %
(l, fl.size(i)), f)
except error.CensoredNodeError:
if ui.config("censor", "policy", "abort") == "abort":
err(lr, _("censored file data"), f)
except Exception as inst:
exc(lr, _("unpacking %s") % short(n), inst, f)
# check renames
try:
if rp:
if lr is not None and ui.verbose:
ctx = lrugetctx(lr)
found = False
for pctx in ctx.parents():
if rp[0] in pctx:
found = True
break
if not found:
warn(_("warning: copy source of '%s' not"
" in parents of %s") % (f, ctx))
fl2 = repo.file(rp[0])
if not len(fl2):
err(lr, _("empty or missing copy source revlog %s:%s")
% (rp[0], short(rp[1])), f)
elif rp[1] == nullid:
ui.note(_("warning: %s@%s: copy source"
" revision is nullid %s:%s\n")
% (f, lr, rp[0], short(rp[1])))
else:
fl2.rev(rp[1])
except Exception as inst:
exc(lr, _("checking rename of %s") % short(n), inst, f)
# cross-check
if f in filenodes:
fns = [(lr, n) for n, lr in filenodes[f].iteritems()]
for lr, node in sorted(fns):
err(lr, _("%s in manifests not found") % short(node), f)
ui.progress(_('checking'), None)
for f in storefiles:
warn(_("warning: orphan revlog '%s'") % f)
ui.status(_("%d files, %d changesets, %d total revisions\n") %
(len(files), len(cl), revisions))
if warnings[0]:
ui.warn(_("%d warnings encountered!\n") % warnings[0])
if fncachewarned:
ui.warn(_('hint: run "hg debugrebuildfncache" to recover from '
'corrupt fncache\n'))
if errors[0]:
ui.warn(_("%d integrity errors encountered!\n") % errors[0])
if badrevs:
ui.warn(_("(first damaged changeset appears to be %d)\n")
% min(badrevs))
return 1
def annotate(self, follow=False, linenumber=None):
"""returns a list of tuples of (ctx, line) for each line
in the file, where ctx is the filectx of the node where
that line was last changed.
This returns tuples of ((ctx, linenumber), line) for each line,
if "linenumber" parameter is NOT "None".
In such tuples, linenumber means one at the first appearance
in the managed file.
To reduce annotation cost,
this returns fixed value(False is used) as linenumber,
if "linenumber" parameter is "False"."""
def decorate_compat(text, rev):
return ([rev] * len(text.splitlines()), text)
def without_linenumber(text, rev):
return ([(rev, False)] * len(text.splitlines()), text)
def with_linenumber(text, rev):
size = len(text.splitlines())
return ([(rev, i) for i in xrange(1, size + 1)], text)
decorate = ((linenumber is None) and decorate_compat) or (linenumber and with_linenumber) or without_linenumber
def pair(parent, child):
for a1, a2, b1, b2 in bdiff.blocks(parent[1], child[1]):
child[0][b1:b2] = parent[0][a1:a2]
return child
getlog = util.lrucachefunc(lambda x: self._repo.file(x))
def getctx(path, fileid):
log = path == self._path and self._filelog or getlog(path)
return filectx(self._repo, path, fileid=fileid, filelog=log)
getctx = util.lrucachefunc(getctx)
def parents(f):
# we want to reuse filectx objects as much as possible
p = f._path
if f._filerev is None: # working dir
pl = [(n.path(), n.filerev()) for n in f.parents()]
else:
pl = [(p, n) for n in f._filelog.parentrevs(f._filerev)]
if follow:
r = f.renamed()
if r:
pl[0] = (r[0], getlog(r[0]).rev(r[1]))
return [getctx(p, n) for p, n in pl if n != nullrev]
# use linkrev to find the first changeset where self appeared
if self.rev() != self.linkrev():
base = self.filectx(self.filerev())
else:
base = self
# This algorithm would prefer to be recursive, but Python is a
# bit recursion-hostile. Instead we do an iterative
# depth-first search.
visit = [base]
hist = {}
pcache = {}
needed = {base: 1}
while visit:
f = visit[-1]
if f not in pcache:
pcache[f] = parents(f)
ready = True
pl = pcache[f]
for p in pl:
if p not in hist:
ready = False
visit.append(p)
needed[p] = needed.get(p, 0) + 1
if ready:
visit.pop()
curr = decorate(f.data(), f)
for p in pl:
curr = pair(hist[p], curr)
if needed[p] == 1:
del hist[p]
else:
needed[p] -= 1
hist[f] = curr
pcache[f] = []
return zip(hist[base][0], hist[base][1].splitlines(True))
def annotate(self, follow=False, linenumber=None, diffopts=None):
'''returns a list of tuples of (ctx, line) for each line
in the file, where ctx is the filectx of the node where
that line was last changed.
This returns tuples of ((ctx, linenumber), line) for each line,
if "linenumber" parameter is NOT "None".
In such tuples, linenumber means one at the first appearance
in the managed file.
To reduce annotation cost,
this returns fixed value(False is used) as linenumber,
if "linenumber" parameter is "False".'''
def decorate_compat(text, rev):
return ([rev] * len(text.splitlines()), text)
def without_linenumber(text, rev):
return ([(rev, False)] * len(text.splitlines()), text)
def with_linenumber(text, rev):
size = len(text.splitlines())
return ([(rev, i) for i in xrange(1, size + 1)], text)
decorate = (((linenumber is None) and decorate_compat)
or (linenumber and with_linenumber) or without_linenumber)
def pair(parent, child):
blocks = mdiff.allblocks(parent[1],
child[1],
opts=diffopts,
refine=True)
for (a1, a2, b1, b2), t in blocks:
# Changed blocks ('!') or blocks made only of blank lines ('~')
# belong to the child.
if t == '=':
child[0][b1:b2] = parent[0][a1:a2]
return child
getlog = util.lrucachefunc(lambda x: self._repo.file(x))
def parents(f):
pl = f.parents()
# Don't return renamed parents if we aren't following.
if not follow:
pl = [p for p in pl if p.path() == f.path()]
# renamed filectx won't have a filelog yet, so set it
# from the cache to save time
for p in pl:
if not '_filelog' in p.__dict__:
p._filelog = getlog(p.path())
return pl
# use linkrev to find the first changeset where self appeared
if self.rev() != self.linkrev():
base = self.filectx(self.filenode())
else:
base = self
# This algorithm would prefer to be recursive, but Python is a
# bit recursion-hostile. Instead we do an iterative
# depth-first search.
visit = [base]
hist = {}
pcache = {}
needed = {base: 1}
while visit:
f = visit[-1]
pcached = f in pcache
if not pcached:
pcache[f] = parents(f)
ready = True
pl = pcache[f]
for p in pl:
if p not in hist:
ready = False
visit.append(p)
if not pcached:
needed[p] = needed.get(p, 0) + 1
if ready:
visit.pop()
reusable = f in hist
if reusable:
curr = hist[f]
else:
curr = decorate(f.data(), f)
for p in pl:
if not reusable:
curr = pair(hist[p], curr)
if needed[p] == 1:
del hist[p]
del needed[p]
else:
needed[p] -= 1
hist[f] = curr
pcache[f] = []
return zip(hist[base][0], hist[base][1].splitlines(True))
def copies(repo, c1, c2, ca, checkdirs=False):
"""
Find moves and copies between context c1 and c2
"""
# avoid silly behavior for update from empty dir
if not c1 or not c2 or c1 == c2:
return {}, {}
# avoid silly behavior for parent -> working dir
if c2.node() is None and c1.node() == repo.dirstate.parents()[0]:
return repo.dirstate.copies(), {}
limit = _findlimit(repo, c1.rev(), c2.rev())
m1 = c1.manifest()
m2 = c2.manifest()
ma = ca.manifest()
def makectx(f, n):
if len(n) != 20: # in a working context?
if c1.rev() is None:
return c1.filectx(f)
return c2.filectx(f)
return repo.filectx(f, fileid=n)
ctx = util.lrucachefunc(makectx)
copy = {}
fullcopy = {}
diverge = {}
def checkcopies(f, m1, m2):
'''check possible copies of f from m1 to m2'''
c1 = ctx(f, m1[f])
for of in _findoldnames(c1, limit):
fullcopy[f] = of # remember for dir rename detection
if of in m2: # original file not in other manifest?
# if the original file is unchanged on the other branch,
# no merge needed
if m2[of] != ma.get(of):
c2 = ctx(of, m2[of])
ca = c1.ancestor(c2)
# related and named changed on only one side?
if ca and (ca.path() == f or ca.path() == c2.path()):
if c1 != ca or c2 != ca: # merge needed?
copy[f] = of
elif of in ma:
diverge.setdefault(of, []).append(f)
repo.ui.debug(_(" searching for copies back to rev %d\n") % limit)
u1 = _nonoverlap(m1, m2, ma)
u2 = _nonoverlap(m2, m1, ma)
if u1:
repo.ui.debug(_(" unmatched files in local:\n %s\n")
% "\n ".join(u1))
if u2:
repo.ui.debug(_(" unmatched files in other:\n %s\n")
% "\n ".join(u2))
for f in u1:
checkcopies(f, m1, m2)
for f in u2:
checkcopies(f, m2, m1)
diverge2 = set()
for of, fl in diverge.items():
if len(fl) == 1:
del diverge[of] # not actually divergent
else:
diverge2.update(fl) # reverse map for below
if fullcopy:
repo.ui.debug(_(" all copies found (* = to merge, ! = divergent):\n"))
for f in fullcopy:
note = ""
if f in copy: note += "*"
if f in diverge2: note += "!"
repo.ui.debug(" %s -> %s %s\n" % (f, fullcopy[f], note))
del diverge2
if not fullcopy or not checkdirs:
return copy, diverge
repo.ui.debug(_(" checking for directory renames\n"))
# generate a directory move map
d1, d2 = _dirs(m1), _dirs(m2)
invalid = set()
dirmove = {}
# examine each file copy for a potential directory move, which is
# when all the files in a directory are moved to a new directory
for dst, src in fullcopy.iteritems():
dsrc, ddst = _dirname(src), _dirname(dst)
if dsrc in invalid:
# already seen to be uninteresting
continue
elif dsrc in d1 and ddst in d1:
# directory wasn't entirely moved locally
invalid.add(dsrc)
elif dsrc in d2 and ddst in d2:
# directory wasn't entirely moved remotely
invalid.add(dsrc)
elif dsrc in dirmove and dirmove[dsrc] != ddst:
# files from the same directory moved to two different places
invalid.add(dsrc)
else:
# looks good so far
dirmove[dsrc + "/"] = ddst + "/"
for i in invalid:
if i in dirmove:
del dirmove[i]
del d1, d2, invalid
if not dirmove:
return copy, diverge
for d in dirmove:
repo.ui.debug(_(" dir %s -> %s\n") % (d, dirmove[d]))
# check unaccounted nonoverlapping files against directory moves
for f in u1 + u2:
if f not in fullcopy:
for d in dirmove:
if f.startswith(d):
# new file added in a directory that was moved, move it
df = dirmove[d] + f[len(d):]
if df not in copy:
copy[f] = df
repo.ui.debug(_(" file %s -> %s\n") % (f, copy[f]))
break
return copy, diverge
def _verify(repo):
repo = repo.unfiltered()
mflinkrevs = {}
filelinkrevs = {}
filenodes = {}
revisions = 0
badrevs = set()
errors = [0]
warnings = [0]
ui = repo.ui
cl = repo.changelog
mf = repo.manifest
lrugetctx = util.lrucachefunc(repo.changectx)
if not repo.url().startswith('file:'):
raise util.Abort(_("cannot verify bundle or remote repos"))
def err(linkrev, msg, filename=None):
if linkrev is not None:
badrevs.add(linkrev)
else:
linkrev = '?'
msg = "%s: %s" % (linkrev, msg)
if filename:
msg = "%s@%s" % (filename, msg)
ui.warn(" " + msg + "\n")
errors[0] += 1
def exc(linkrev, msg, inst, filename=None):
if isinstance(inst, KeyboardInterrupt):
ui.warn(_("interrupted"))
raise
if not str(inst):
inst = repr(inst)
err(linkrev, "%s: %s" % (msg, inst), filename)
def warn(msg):
ui.warn(msg + "\n")
warnings[0] += 1
def checklog(obj, name, linkrev):
if not len(obj) and (havecl or havemf):
err(linkrev, _("empty or missing %s") % name)
return
d = obj.checksize()
if d[0]:
err(None, _("data length off by %d bytes") % d[0], name)
if d[1]:
err(None, _("index contains %d extra bytes") % d[1], name)
if obj.version != revlog.REVLOGV0:
if not revlogv1:
warn(_("warning: `%s' uses revlog format 1") % name)
elif revlogv1:
warn(_("warning: `%s' uses revlog format 0") % name)
def checkentry(obj, i, node, seen, linkrevs, f):
lr = obj.linkrev(obj.rev(node))
if lr < 0 or (havecl and lr not in linkrevs):
if lr < 0 or lr >= len(cl):
msg = _("rev %d points to nonexistent changeset %d")
else:
msg = _("rev %d points to unexpected changeset %d")
err(None, msg % (i, lr), f)
if linkrevs:
if f and len(linkrevs) > 1:
try:
# attempt to filter down to real linkrevs
linkrevs = [l for l in linkrevs
if lrugetctx(l)[f].filenode() == node]
except Exception:
pass
warn(_(" (expected %s)") % " ".join(map(str, linkrevs)))
lr = None # can't be trusted
try:
p1, p2 = obj.parents(node)
if p1 not in seen and p1 != nullid:
err(lr, _("unknown parent 1 %s of %s") %
(short(p1), short(node)), f)
if p2 not in seen and p2 != nullid:
err(lr, _("unknown parent 2 %s of %s") %
(short(p2), short(node)), f)
except Exception, inst:
exc(lr, _("checking parents of %s") % short(node), inst, f)
if node in seen:
err(lr, _("duplicate revision %d (%d)") % (i, seen[node]), f)
seen[node] = i
return lr
def annotate(self, follow=False, linenumber=None):
'''returns a list of tuples of (ctx, line) for each line
in the file, where ctx is the filectx of the node where
that line was last changed.
This returns tuples of ((ctx, linenumber), line) for each line,
if "linenumber" parameter is NOT "None".
In such tuples, linenumber means one at the first appearance
in the managed file.
To reduce annotation cost,
this returns fixed value(False is used) as linenumber,
if "linenumber" parameter is "False".'''
def decorate_compat(text, rev):
return ([rev] * len(text.splitlines()), text)
def without_linenumber(text, rev):
return ([(rev, False)] * len(text.splitlines()), text)
def with_linenumber(text, rev):
size = len(text.splitlines())
return ([(rev, i) for i in xrange(1, size + 1)], text)
decorate = (((linenumber is None) and decorate_compat) or
(linenumber and with_linenumber) or
without_linenumber)
def pair(parent, child):
for a1, a2, b1, b2 in bdiff.blocks(parent[1], child[1]):
child[0][b1:b2] = parent[0][a1:a2]
return child
getlog = util.lrucachefunc(lambda x: self._repo.file(x))
def getctx(path, fileid):
log = path == self._path and self._filelog or getlog(path)
return filectx(self._repo, path, fileid=fileid, filelog=log)
getctx = util.lrucachefunc(getctx)
def parents(f):
# we want to reuse filectx objects as much as possible
p = f._path
if f._filerev is None: # working dir
pl = [(n.path(), n.filerev()) for n in f.parents()]
else:
pl = [(p, n) for n in f._filelog.parentrevs(f._filerev)]
if follow:
r = f.renamed()
if r:
pl[0] = (r[0], getlog(r[0]).rev(r[1]))
return [getctx(p, n) for p, n in pl if n != nullrev]
# use linkrev to find the first changeset where self appeared
if self.rev() != self.linkrev():
base = self.filectx(self.filerev())
else:
base = self
# find all ancestors
needed = {base: 1}
visit = [base]
files = [base._path]
while visit:
f = visit.pop(0)
for p in parents(f):
if p not in needed:
needed[p] = 1
visit.append(p)
if p._path not in files:
files.append(p._path)
else:
# count how many times we'll use this
needed[p] += 1
# sort by revision (per file) which is a topological order
visit = []
for f in files:
visit.extend(n for n in needed if n._path == f)
hist = {}
for f in sorted(visit, key=lambda x: x.rev()):
curr = decorate(f.data(), f)
for p in parents(f):
curr = pair(hist[p], curr)
# trim the history of unneeded revs
needed[p] -= 1
if not needed[p]:
del hist[p]
hist[f] = curr
return zip(hist[f][0], hist[f][1].splitlines(True))
def colored(dag, repo):
"""annotates a DAG with colored edge information
For each DAG node this function emits tuples::
(id, type, data, (col, color), [(col, nextcol, color)])
with the following new elements:
- Tuple (col, color) with column and color index for the current node
- A list of tuples indicating the edges between the current node and its
parents.
"""
seen = []
colors = {}
newcolor = 1
config = {}
for key, val in repo.ui.configitems('graph'):
if '.' in key:
branch, setting = key.rsplit('.', 1)
# Validation
if setting == "width" and val.isdigit():
config.setdefault(branch, {})[setting] = int(val)
elif setting == "color" and val.isalnum():
config.setdefault(branch, {})[setting] = val
if config:
getconf = util.lrucachefunc(
lambda rev: config.get(repo[rev].branch(), {}))
else:
getconf = lambda rev: {}
for (cur, type, data, parents) in dag:
# Compute seen and next
if cur not in seen:
seen.append(cur) # new head
colors[cur] = newcolor
newcolor += 1
col = seen.index(cur)
color = colors.pop(cur)
next = seen[:]
# Add parents to next
addparents = [p for p in parents if p not in next]
next[col:col + 1] = addparents
# Set colors for the parents
for i, p in enumerate(addparents):
if not i:
colors[p] = color
else:
colors[p] = newcolor
newcolor += 1
# Add edges to the graph
edges = []
for ecol, eid in enumerate(seen):
if eid in next:
bconf = getconf(eid)
edges.append((ecol, next.index(eid), colors[eid],
bconf.get('width', -1), bconf.get('color', '')))
elif eid == cur:
for p in parents:
bconf = getconf(p)
edges.append(
(ecol, next.index(p), color, bconf.get('width', -1),
bconf.get('color', '')))
# Yield and move on
yield (cur, type, data, (col, color), edges)
seen = next
def copies(repo, c1, c2, ca, checkdirs=False):
"""
Find moves and copies between context c1 and c2
"""
# avoid silly behavior for update from empty dir
if not c1 or not c2 or c1 == c2:
return {}, {}
# avoid silly behavior for parent -> working dir
if c2.node() is None and c1.node() == repo.dirstate.parents()[0]:
return repo.dirstate.copies(), {}
limit = _findlimit(repo, c1.rev(), c2.rev())
if limit is None:
# no common ancestor, no copies
return {}, {}
m1 = c1.manifest()
m2 = c2.manifest()
ma = ca.manifest()
def makectx(f, n):
if len(n) != 20: # in a working context?
if c1.rev() is None:
return c1.filectx(f)
return c2.filectx(f)
return repo.filectx(f, fileid=n)
ctx = util.lrucachefunc(makectx)
copy = {}
fullcopy = {}
diverge = {}
def related(f1, f2, limit):
# Walk back to common ancestor to see if the two files originate
# from the same file. Since workingfilectx's rev() is None it messes
# up the integer comparison logic, hence the pre-step check for
# None (f1 and f2 can only be workingfilectx's initially).
if f1 == f2:
return f1 # a match
g1, g2 = f1.ancestors(), f2.ancestors()
try:
f1r, f2r = f1.rev(), f2.rev()
if f1r is None:
f1 = g1.next()
if f2r is None:
f2 = g2.next()
while 1:
f1r, f2r = f1.rev(), f2.rev()
if f1r > f2r:
f1 = g1.next()
elif f2r > f1r:
f2 = g2.next()
elif f1 == f2:
return f1 # a match
elif f1r == f2r or f1r < limit or f2r < limit:
return False # copy no longer relevant
except StopIteration:
return False
def checkcopies(f, m1, m2):
'''check possible copies of f from m1 to m2'''
of = None
seen = set([f])
for oc in ctx(f, m1[f]).ancestors():
ocr = oc.rev()
of = oc.path()
if of in seen:
# check limit late - grab last rename before
if ocr < limit:
break
continue
seen.add(of)
fullcopy[f] = of # remember for dir rename detection
if of not in m2:
continue # no match, keep looking
if m2[of] == ma.get(of):
break # no merge needed, quit early
c2 = ctx(of, m2[of])
cr = related(oc, c2, ca.rev())
if cr and (of == f or of == c2.path()): # non-divergent
copy[f] = of
of = None
break
if of in ma:
diverge.setdefault(of, []).append(f)
repo.ui.debug(" searching for copies back to rev %d\n" % limit)
u1 = _nonoverlap(m1, m2, ma)
u2 = _nonoverlap(m2, m1, ma)
if u1:
repo.ui.debug(" unmatched files in local:\n %s\n" %
"\n ".join(u1))
if u2:
repo.ui.debug(" unmatched files in other:\n %s\n" %
"\n ".join(u2))
for f in u1:
checkcopies(f, m1, m2)
for f in u2:
checkcopies(f, m2, m1)
diverge2 = set()
for of, fl in diverge.items():
if len(fl) == 1 or of in c2:
del diverge[of] # not actually divergent, or not a rename
else:
diverge2.update(fl) # reverse map for below
if fullcopy:
repo.ui.debug(" all copies found (* = to merge, ! = divergent):\n")
for f in fullcopy:
note = ""
if f in copy:
note += "*"
if f in diverge2:
note += "!"
repo.ui.debug(" %s -> %s %s\n" % (f, fullcopy[f], note))
del diverge2
if not fullcopy or not checkdirs:
return copy, diverge
repo.ui.debug(" checking for directory renames\n")
# generate a directory move map
d1, d2 = _dirs(m1), _dirs(m2)
invalid = set()
dirmove = {}
# examine each file copy for a potential directory move, which is
# when all the files in a directory are moved to a new directory
for dst, src in fullcopy.iteritems():
dsrc, ddst = _dirname(src), _dirname(dst)
if dsrc in invalid:
# already seen to be uninteresting
continue
elif dsrc in d1 and ddst in d1:
# directory wasn't entirely moved locally
invalid.add(dsrc)
elif dsrc in d2 and ddst in d2:
# directory wasn't entirely moved remotely
invalid.add(dsrc)
elif dsrc in dirmove and dirmove[dsrc] != ddst:
# files from the same directory moved to two different places
invalid.add(dsrc)
else:
# looks good so far
dirmove[dsrc + "/"] = ddst + "/"
for i in invalid:
if i in dirmove:
del dirmove[i]
del d1, d2, invalid
if not dirmove:
return copy, diverge
for d in dirmove:
repo.ui.debug(" dir %s -> %s\n" % (d, dirmove[d]))
# check unaccounted nonoverlapping files against directory moves
for f in u1 + u2:
if f not in fullcopy:
for d in dirmove:
if f.startswith(d):
# new file added in a directory that was moved, move it
df = dirmove[d] + f[len(d):]
if df not in copy:
copy[f] = df
repo.ui.debug(" file %s -> %s\n" % (f, copy[f]))
break
return copy, diverge
def annotate(self, follow=False, linenumber=None, diffopts=None):
'''returns a list of tuples of (ctx, line) for each line
in the file, where ctx is the filectx of the node where
that line was last changed.
This returns tuples of ((ctx, linenumber), line) for each line,
if "linenumber" parameter is NOT "None".
In such tuples, linenumber means one at the first appearance
in the managed file.
To reduce annotation cost,
this returns fixed value(False is used) as linenumber,
if "linenumber" parameter is "False".'''
def decorate_compat(text, rev):
return ([rev] * len(text.splitlines()), text)
def without_linenumber(text, rev):
return ([(rev, False)] * len(text.splitlines()), text)
def with_linenumber(text, rev):
size = len(text.splitlines())
return ([(rev, i) for i in xrange(1, size + 1)], text)
decorate = (((linenumber is None) and decorate_compat) or
(linenumber and with_linenumber) or
without_linenumber)
def pair(parent, child):
blocks = mdiff.allblocks(parent[1], child[1], opts=diffopts,
refine=True)
for (a1, a2, b1, b2), t in blocks:
# Changed blocks ('!') or blocks made only of blank lines ('~')
# belong to the child.
if t == '=':
child[0][b1:b2] = parent[0][a1:a2]
return child
getlog = util.lrucachefunc(lambda x: self._repo.file(x))
def parents(f):
pl = f.parents()
# Don't return renamed parents if we aren't following.
if not follow:
pl = [p for p in pl if p.path() == f.path()]
# renamed filectx won't have a filelog yet, so set it
# from the cache to save time
for p in pl:
if not '_filelog' in p.__dict__:
p._filelog = getlog(p.path())
return pl
# use linkrev to find the first changeset where self appeared
if self.rev() != self.linkrev():
base = self.filectx(self.filenode())
else:
base = self
# This algorithm would prefer to be recursive, but Python is a
# bit recursion-hostile. Instead we do an iterative
# depth-first search.
visit = [base]
hist = {}
pcache = {}
needed = {base: 1}
while visit:
f = visit[-1]
pcached = f in pcache
if not pcached:
pcache[f] = parents(f)
ready = True
pl = pcache[f]
for p in pl:
if p not in hist:
ready = False
visit.append(p)
if not pcached:
needed[p] = needed.get(p, 0) + 1
if ready:
visit.pop()
reusable = f in hist
if reusable:
curr = hist[f]
else:
curr = decorate(f.data(), f)
for p in pl:
if not reusable:
curr = pair(hist[p], curr)
if needed[p] == 1:
del hist[p]
del needed[p]
else:
needed[p] -= 1
hist[f] = curr
pcache[f] = []
return zip(hist[base][0], hist[base][1].splitlines(True))
def mergecopies(repo, c1, c2, ca):
"""
Find moves and copies between context c1 and c2 that are relevant
for merging.
Returns four dicts: "copy", "movewithdir", "diverge", and
"renamedelete".
"copy" is a mapping from destination name -> source name,
where source is in c1 and destination is in c2 or vice-versa.
"movewithdir" is a mapping from source name -> destination name,
where the file at source present in one context but not the other
needs to be moved to destination by the merge process, because the
other context moved the directory it is in.
"diverge" is a mapping of source name -> list of destination names
for divergent renames.
"renamedelete" is a mapping of source name -> list of destination
names for files deleted in c1 that were renamed in c2 or vice-versa.
"""
# avoid silly behavior for update from empty dir
if not c1 or not c2 or c1 == c2:
return {}, {}, {}, {}
# avoid silly behavior for parent -> working dir
if c2.node() is None and c1.node() == repo.dirstate.p1():
return repo.dirstate.copies(), {}, {}, {}
limit = _findlimit(repo, c1.rev(), c2.rev())
if limit is None:
# no common ancestor, no copies
return {}, {}, {}, {}
m1 = c1.manifest()
m2 = c2.manifest()
ma = ca.manifest()
def makectx(f, n):
if len(n) != 20: # in a working context?
if c1.rev() is None:
return c1.filectx(f)
return c2.filectx(f)
return repo.filectx(f, fileid=n)
ctx = util.lrucachefunc(makectx)
copy = {}
movewithdir = {}
fullcopy = {}
diverge = {}
repo.ui.debug(" searching for copies back to rev %d\n" % limit)
u1 = _nonoverlap(m1, m2, ma)
u2 = _nonoverlap(m2, m1, ma)
if u1:
repo.ui.debug(" unmatched files in local:\n %s\n" %
"\n ".join(u1))
if u2:
repo.ui.debug(" unmatched files in other:\n %s\n" %
"\n ".join(u2))
for f in u1:
checkcopies(ctx, f, m1, m2, ca, limit, diverge, copy, fullcopy)
for f in u2:
checkcopies(ctx, f, m2, m1, ca, limit, diverge, copy, fullcopy)
renamedelete = {}
renamedelete2 = set()
diverge2 = set()
for of, fl in diverge.items():
if len(fl) == 1 or of in c1 or of in c2:
del diverge[of] # not actually divergent, or not a rename
if of not in c1 and of not in c2:
# renamed on one side, deleted on the other side, but filter
# out files that have been renamed and then deleted
renamedelete[of] = [f for f in fl if f in c1 or f in c2]
renamedelete2.update(fl) # reverse map for below
else:
diverge2.update(fl) # reverse map for below
bothnew = sorted([d for d in m1 if d in m2 and d not in ma])
if bothnew:
repo.ui.debug(" unmatched files new in both:\n %s\n" %
"\n ".join(bothnew))
bothdiverge, _copy, _fullcopy = {}, {}, {}
for f in bothnew:
checkcopies(ctx, f, m1, m2, ca, limit, bothdiverge, _copy, _fullcopy)
checkcopies(ctx, f, m2, m1, ca, limit, bothdiverge, _copy, _fullcopy)
for of, fl in bothdiverge.items():
if len(fl) == 2 and fl[0] == fl[1]:
copy[fl[0]] = of # not actually divergent, just matching renames
if fullcopy and repo.ui.debugflag:
repo.ui.debug(" all copies found (* = to merge, ! = divergent, "
"% = renamed and deleted):\n")
for f in sorted(fullcopy):
note = ""
if f in copy:
note += "*"
if f in diverge2:
note += "!"
if f in renamedelete2:
note += "%"
repo.ui.debug(" src: '%s' -> dst: '%s' %s\n" %
(fullcopy[f], f, note))
del diverge2
if not fullcopy:
return copy, movewithdir, diverge, renamedelete
repo.ui.debug(" checking for directory renames\n")
# generate a directory move map
d1, d2 = c1.dirs(), c2.dirs()
d1.addpath('/')
d2.addpath('/')
invalid = set()
dirmove = {}
# examine each file copy for a potential directory move, which is
# when all the files in a directory are moved to a new directory
for dst, src in fullcopy.iteritems():
dsrc, ddst = _dirname(src), _dirname(dst)
if dsrc in invalid:
# already seen to be uninteresting
continue
elif dsrc in d1 and ddst in d1:
# directory wasn't entirely moved locally
invalid.add(dsrc)
elif dsrc in d2 and ddst in d2:
# directory wasn't entirely moved remotely
invalid.add(dsrc)
elif dsrc in dirmove and dirmove[dsrc] != ddst:
# files from the same directory moved to two different places
invalid.add(dsrc)
else:
# looks good so far
dirmove[dsrc + "/"] = ddst + "/"
for i in invalid:
if i in dirmove:
del dirmove[i]
del d1, d2, invalid
if not dirmove:
return copy, movewithdir, diverge, renamedelete
for d in dirmove:
repo.ui.debug(" discovered dir src: '%s' -> dst: '%s'\n" %
(d, dirmove[d]))
# check unaccounted nonoverlapping files against directory moves
for f in u1 + u2:
if f not in fullcopy:
for d in dirmove:
if f.startswith(d):
# new file added in a directory that was moved, move it
df = dirmove[d] + f[len(d):]
if df not in copy:
movewithdir[f] = df
repo.ui.debug((" pending file src: '%s' -> "
"dst: '%s'\n") % (f, df))
break
return copy, movewithdir, diverge, renamedelete
def colored(dag, repo):
"""annotates a DAG with colored edge information
For each DAG node this function emits tuples::
(id, type, data, (col, color), [(col, nextcol, color)])
with the following new elements:
- Tuple (col, color) with column and color index for the current node
- A list of tuples indicating the edges between the current node and its
parents.
"""
seen = []
colors = {}
newcolor = 1
config = {}
for key, val in repo.ui.configitems('graph'):
if '.' in key:
branch, setting = key.rsplit('.', 1)
# Validation
if setting == "width" and val.isdigit():
config.setdefault(branch, {})[setting] = int(val)
elif setting == "color" and val.isalnum():
config.setdefault(branch, {})[setting] = val
if config:
getconf = util.lrucachefunc(
lambda rev: config.get(repo[rev].branch(), {}))
else:
getconf = lambda rev: {}
for (cur, type, data, parents) in dag:
# Compute seen and next
if cur not in seen:
seen.append(cur) # new head
colors[cur] = newcolor
newcolor += 1
col = seen.index(cur)
color = colors.pop(cur)
next = seen[:]
# Add parents to next
addparents = [p for p in parents if p not in next]
next[col:col + 1] = addparents
# Set colors for the parents
for i, p in enumerate(addparents):
if not i:
colors[p] = color
else:
colors[p] = newcolor
newcolor += 1
# Add edges to the graph
edges = []
for ecol, eid in enumerate(seen):
if eid in next:
bconf = getconf(eid)
edges.append((
ecol, next.index(eid), colors[eid],
bconf.get('width', -1),
bconf.get('color', '')))
elif eid == cur:
for p in parents:
bconf = getconf(p)
edges.append((
ecol, next.index(p), color,
bconf.get('width', -1),
bconf.get('color', '')))
# Yield and move on
yield (cur, type, data, (col, color), edges)
seen = next
def copies(repo, c1, c2, ca, checkdirs=False):
"""
Find moves and copies between context c1 and c2
"""
# avoid silly behavior for update from empty dir
if not c1 or not c2 or c1 == c2:
return {}, {}
# avoid silly behavior for parent -> working dir
if c2.node() is None and c1.node() == repo.dirstate.parents()[0]:
return repo.dirstate.copies(), {}
limit = _findlimit(repo, c1.rev(), c2.rev())
if limit is None:
# no common ancestor, no copies
return {}, {}
m1 = c1.manifest()
m2 = c2.manifest()
ma = ca.manifest()
def makectx(f, n):
if len(n) != 20: # in a working context?
if c1.rev() is None:
return c1.filectx(f)
return c2.filectx(f)
return repo.filectx(f, fileid=n)
ctx = util.lrucachefunc(makectx)
copy = {}
fullcopy = {}
diverge = {}
def related(f1, f2, limit):
# Walk back to common ancestor to see if the two files originate
# from the same file. Since workingfilectx's rev() is None it messes
# up the integer comparison logic, hence the pre-step check for
# None (f1 and f2 can only be workingfilectx's initially).
if f1 == f2:
return f1 # a match
g1, g2 = f1.ancestors(), f2.ancestors()
try:
f1r, f2r = f1.rev(), f2.rev()
if f1r is None:
f1 = g1.next()
if f2r is None:
f2 = g2.next()
while 1:
f1r, f2r = f1.rev(), f2.rev()
if f1r > f2r:
f1 = g1.next()
elif f2r > f1r:
f2 = g2.next()
elif f1 == f2:
return f1 # a match
elif f1r == f2r or f1r < limit or f2r < limit:
return False # copy no longer relevant
except StopIteration:
return False
def checkcopies(f, m1, m2):
'''check possible copies of f from m1 to m2'''
of = None
seen = set([f])
for oc in ctx(f, m1[f]).ancestors():
ocr = oc.rev()
of = oc.path()
if of in seen:
# check limit late - grab last rename before
if ocr < limit:
break
continue
seen.add(of)
fullcopy[f] = of # remember for dir rename detection
if of not in m2:
continue # no match, keep looking
if m2[of] == ma.get(of):
break # no merge needed, quit early
c2 = ctx(of, m2[of])
cr = related(oc, c2, ca.rev())
if cr and (of == f or of == c2.path()): # non-divergent
copy[f] = of
of = None
break
if of in ma:
diverge.setdefault(of, []).append(f)
repo.ui.debug(" searching for copies back to rev %d\n" % limit)
u1 = _nonoverlap(m1, m2, ma)
u2 = _nonoverlap(m2, m1, ma)
if u1:
repo.ui.debug(" unmatched files in local:\n %s\n"
% "\n ".join(u1))
if u2:
repo.ui.debug(" unmatched files in other:\n %s\n"
% "\n ".join(u2))
for f in u1:
checkcopies(f, m1, m2)
for f in u2:
checkcopies(f, m2, m1)
diverge2 = set()
for of, fl in diverge.items():
if len(fl) == 1 or of in c2:
del diverge[of] # not actually divergent, or not a rename
else:
diverge2.update(fl) # reverse map for below
if fullcopy:
repo.ui.debug(" all copies found (* = to merge, ! = divergent):\n")
for f in fullcopy:
note = ""
if f in copy:
note += "*"
if f in diverge2:
note += "!"
repo.ui.debug(" %s -> %s %s\n" % (f, fullcopy[f], note))
del diverge2
if not fullcopy or not checkdirs:
return copy, diverge
repo.ui.debug(" checking for directory renames\n")
# generate a directory move map
d1, d2 = _dirs(m1), _dirs(m2)
invalid = set()
dirmove = {}
# examine each file copy for a potential directory move, which is
# when all the files in a directory are moved to a new directory
for dst, src in fullcopy.iteritems():
dsrc, ddst = _dirname(src), _dirname(dst)
if dsrc in invalid:
# already seen to be uninteresting
continue
elif dsrc in d1 and ddst in d1:
# directory wasn't entirely moved locally
invalid.add(dsrc)
elif dsrc in d2 and ddst in d2:
# directory wasn't entirely moved remotely
invalid.add(dsrc)
elif dsrc in dirmove and dirmove[dsrc] != ddst:
# files from the same directory moved to two different places
invalid.add(dsrc)
else:
# looks good so far
dirmove[dsrc + "/"] = ddst + "/"
for i in invalid:
if i in dirmove:
del dirmove[i]
del d1, d2, invalid
if not dirmove:
return copy, diverge
for d in dirmove:
repo.ui.debug(" dir %s -> %s\n" % (d, dirmove[d]))
# check unaccounted nonoverlapping files against directory moves
for f in u1 + u2:
if f not in fullcopy:
for d in dirmove:
if f.startswith(d):
# new file added in a directory that was moved, move it
df = dirmove[d] + f[len(d):]
if df not in copy:
copy[f] = df
repo.ui.debug(" file %s -> %s\n" % (f, copy[f]))
break
return copy, diverge
def annotate(self, follow=False, linenumber=None):
'''returns a list of tuples of (ctx, line) for each line
in the file, where ctx is the filectx of the node where
that line was last changed.
This returns tuples of ((ctx, linenumber), line) for each line,
if "linenumber" parameter is NOT "None".
In such tuples, linenumber means one at the first appearance
in the managed file.
To reduce annotation cost,
this returns fixed value(False is used) as linenumber,
if "linenumber" parameter is "False".'''
def decorate_compat(text, rev):
return ([rev] * len(text.splitlines()), text)
def without_linenumber(text, rev):
return ([(rev, False)] * len(text.splitlines()), text)
def with_linenumber(text, rev):
size = len(text.splitlines())
return ([(rev, i) for i in xrange(1, size + 1)], text)
decorate = (((linenumber is None) and decorate_compat) or
(linenumber and with_linenumber) or
without_linenumber)
def pair(parent, child):
for a1, a2, b1, b2 in bdiff.blocks(parent[1], child[1]):
child[0][b1:b2] = parent[0][a1:a2]
return child
getlog = util.lrucachefunc(lambda x: self._repo.file(x))
def getctx(path, fileid):
log = path == self._path and self._filelog or getlog(path)
return filectx(self._repo, path, fileid=fileid, filelog=log)
getctx = util.lrucachefunc(getctx)
def parents(f):
# we want to reuse filectx objects as much as possible
p = f._path
if f._filerev is None: # working dir
pl = [(n.path(), n.filerev()) for n in f.parents()]
else:
pl = [(p, n) for n in f._filelog.parentrevs(f._filerev)]
if follow:
r = f.renamed()
if r:
pl[0] = (r[0], getlog(r[0]).rev(r[1]))
return [getctx(p, n) for p, n in pl if n != nullrev]
# use linkrev to find the first changeset where self appeared
if self.rev() != self.linkrev():
base = self.filectx(self.filerev())
else:
base = self
# find all ancestors
needed = {base: 1}
visit = [base]
files = [base._path]
while visit:
f = visit.pop(0)
for p in parents(f):
if p not in needed:
needed[p] = 1
visit.append(p)
if p._path not in files:
files.append(p._path)
else:
# count how many times we'll use this
needed[p] += 1
# sort by revision (per file) which is a topological order
visit = []
for f in files:
visit.extend(n for n in needed if n._path == f)
hist = {}
for f in sorted(visit, key=lambda x: x.rev()):
curr = decorate(f.data(), f)
for p in parents(f):
curr = pair(hist[p], curr)
# trim the history of unneeded revs
needed[p] -= 1
if not needed[p]:
del hist[p]
hist[f] = curr
return zip(hist[f][0], hist[f][1].splitlines(True))
