def stream_out_shallow(repo, proto, other):
includepattern = None
excludepattern = None
raw = other.get("includepattern")
if raw:
includepattern = raw.split("\0")
raw = other.get("excludepattern")
if raw:
excludepattern = raw.split("\0")
oldshallow = state.shallowremote
oldmatch = state.match
oldnoflatmf = state.noflatmf
try:
state.shallowremote = True
state.match = match.always(repo.root, "")
state.noflatmf = other.get("noflatmanifest") == "True"
if includepattern or excludepattern:
state.match = match.match(
repo.root, "", None, includepattern, excludepattern
)
streamres = wireproto.stream(repo, proto)
# Force the first value to execute, so the file list is computed
# within the try/finally scope
first = next(streamres.gen)
second = next(streamres.gen)
def gen():
yield first
yield second
for value in streamres.gen:
yield value
return wireproto.streamres(gen())
finally:
state.shallowremote = oldshallow
state.match = oldmatch
state.noflatmf = oldnoflatmf
def testMatchesCwd(self):
"""Tests matches() on a relpath match with the current directory ('.')
when not in the root directory."""
m = self.parsemanifest(A_DEEPER_MANIFEST)
match = matchmod.match(b"/", b"a/b", [b"."], default=b"relpath")
m2 = m.matches(match)
self.assertEqual(
[
b"a/b/c/bar.py",
b"a/b/c/bar.txt",
b"a/b/c/foo.py",
b"a/b/c/foo.txt",
b"a/b/d/baz.py",
b"a/b/d/qux.py",
b"a/b/d/ten.txt",
b"a/b/dog.py",
b"a/b/fish.py",
],
m2.keys(),
)
def testMatchesDirectory(self):
"""Tests matches() on a relpath match on a directory, which should
match against all files within said directory."""
m = self.parsemanifest(A_DEEPER_MANIFEST)
match = matchmod.match(b"/", b"", [b"a/b"], default=b"relpath")
m2 = m.matches(match)
self.assertEqual(
[
b"a/b/c/bar.py",
b"a/b/c/bar.txt",
b"a/b/c/foo.py",
b"a/b/c/foo.txt",
b"a/b/d/baz.py",
b"a/b/d/qux.py",
b"a/b/d/ten.txt",
b"a/b/dog.py",
b"a/b/fish.py",
],
m2.keys(),
)
def filelogrevset(orig, repo, subset, x):
"""``filelog(pattern)``
Changesets connected to the specified filelog.
For performance reasons, ``filelog()`` does not show every changeset
that affects the requested file(s). See :hg:`help log` for details. For
a slower, more accurate result, use ``file()``.
"""
if not shallowrepo.requirement in repo.requirements:
return orig(repo, subset, x)
# i18n: "filelog" is a keyword
pat = revset.getstring(x, _("filelog requires a pattern"))
m = match.match(repo.root,
repo.getcwd(), [pat],
default="relpath",
ctx=repo[None])
s = set()
if not match.patkind(pat):
# slow
for r in subset:
ctx = repo[r]
cfiles = ctx.files()
for f in m.files():
if f in cfiles:
s.add(ctx.rev())
break
else:
# partial
files = (f for f in repo[None] if m(f))
for f in files:
fctx = repo[None].filectx(f)
s.add(fctx.linkrev())
for actx in fctx.ancestors():
s.add(actx.linkrev())
return smartset.baseset([r for r in subset if r in s])
def generatefiles(self, changedfiles, linknodes, commonrevs, source):
if self._repo.ui.configbool("remotefilelog", "server"):
caps = self._bundlecaps or []
if requirement in caps:
# only send files that don't match the specified patterns
includepattern = None
excludepattern = None
for cap in self._bundlecaps or []:
if cap.startswith("includepattern="):
includepattern = cap[len("includepattern="):].split(
"\0")
elif cap.startswith("excludepattern="):
excludepattern = cap[len("excludepattern="):].split(
"\0")
m = match.always(self._repo.root, "")
if includepattern or excludepattern:
m = match.match(self._repo.root, "", None, includepattern,
excludepattern)
changedfiles = list([f for f in changedfiles if not m(f)])
if requirement in self._repo.requirements:
repo = self._repo
if isinstance(repo, bundlerepo.bundlerepository):
# If the bundle contains filelogs, we can't pull from it, since
# bundlerepo is heavily tied to revlogs. Instead require that
# the user use unbundle instead.
# Force load the filelog data.
bundlerepo.bundlerepository.file(repo, "foo")
if repo._cgfilespos:
raise error.Abort(
"cannot pull from full bundles",
hint="use `hg unbundle` instead",
)
return []
filestosend = self.shouldaddfilegroups(source)
if filestosend == NoFiles:
changedfiles = list(
[f for f in changedfiles if not repo.shallowmatch(f)])
else:
files = []
phasecache = repo._phasecache
cl = repo.changelog
# Prefetch the revisions being bundled
for i, fname in enumerate(sorted(changedfiles)):
filerevlog = repo.file(fname)
linkrevnodes = linknodes(filerevlog, fname)
# Normally we'd prune the linkrevnodes first,
# but that would perform the server fetches one by one.
for fnode, cnode in list(pycompat.iteritems(linkrevnodes)):
# Adjust linknodes so remote file revisions aren't sent
if filestosend == LocalFiles:
if phasecache.phase(
repo, cl.rev(cnode)
) == phases.public and repo.shallowmatch(fname):
del linkrevnodes[fnode]
else:
files.append((fname, hex(fnode)))
else:
files.append((fname, hex(fnode)))
repo.fileservice.prefetch(files)
# Prefetch the revisions that are going to be diffed against
prevfiles = []
for fname, fnode in files:
if repo.shallowmatch(fname):
fnode = bin(fnode)
filerevlog = repo.file(fname)
p1, p2, linknode, copyfrom = filerevlog.getnodeinfo(
fnode)
if p1 != nullid:
prevfiles.append((copyfrom or fname, hex(p1)))
repo.fileservice.prefetch(prevfiles)
return super(shallowcg1packer,
self).generatefiles(changedfiles, linknodes, commonrevs,
source)
def wraprepo(repo):
class shallowrepository(repo.__class__):
@util.propertycache
def name(self):
return self.ui.config("remotefilelog", "reponame", "")
@util.propertycache
def fallbackpath(self):
path = self.ui.config(
"remotefilelog",
"fallbackpath",
# fallbackrepo is the old, deprecated name
self.ui.config("remotefilelog", "fallbackrepo",
self.ui.config("paths", "default")),
)
if not path:
raise error.Abort("no remotefilelog server "
"configured - is your .hg/hgrc trusted?")
return path
@localrepo.unfilteredpropertycache
def fileslog(self):
return remotefilelog.remotefileslog(self)
def maybesparsematch(self, *revs, **kwargs):
"""
A wrapper that allows the remotefilelog to invoke sparsematch() if
this is a sparse repository, or returns None if this is not a
sparse repository.
"""
if util.safehasattr(self, "sparsematch"):
return self.sparsematch(*revs, **kwargs)
return None
def file(self, f):
if f[0] == "/":
f = f[1:]
if self.shallowmatch(f):
return remotefilelog.remotefilelog(self.svfs, f, self)
else:
return super(shallowrepository, self).file(f)
def filectx(self, path, changeid=None, fileid=None):
if self.shallowmatch(path):
return remotefilectx.remotefilectx(self, path, changeid,
fileid)
else:
return super(shallowrepository,
self).filectx(path, changeid, fileid)
@localrepo.unfilteredmethod
def close(self):
result = super(shallowrepository, self).close()
self.fileservice.close()
if "fileslog" in self.__dict__:
self.fileslog.abortpending()
return result
@localrepo.unfilteredmethod
def commitpending(self):
super(shallowrepository, self).commitpending()
self.numtransactioncommits += 1
# In some cases, we can have many transactions in the same repo, in
# which case each one will create a packfile, let's trigger a repack at
# this point to bring the number of packfiles down to a reasonable
# number.
if self.numtransactioncommits >= self.ui.configint(
"remotefilelog", "commitsperrepack"):
domaintenancerepack(self)
self.numtransactioncommits = 0
@localrepo.unfilteredmethod
def commitctx(self, ctx, error=False):
"""Add a new revision to current repository.
Revision information is passed via the context argument.
"""
# some contexts already have manifest nodes, they don't need any
# prefetching (for example if we're just editing a commit message
# we can reuse manifest
if not ctx.manifestnode():
# prefetch files that will likely be compared
m1 = ctx.p1().manifest()
files = []
for f in ctx.modified() + ctx.added():
fparent1 = m1.get(f, nullid)
if fparent1 != nullid:
files.append((f, hex(fparent1)))
self.fileservice.prefetch(files)
return super(shallowrepository, self).commitctx(ctx, error=error)
def backgroundprefetch(self,
revs,
base=None,
repack=False,
pats=None,
opts=None):
"""Runs prefetch in background with optional repack
"""
cmd = [util.hgexecutable(), "-R", self.origroot, "prefetch"]
if repack:
cmd.append("--repack")
if revs:
cmd += ["-r", revs]
if base:
cmd += ["-b", base]
cmd = " ".join(map(util.shellquote, cmd))
runshellcommand(cmd, encoding.environ)
def prefetch(self,
revs,
base=None,
pats=None,
opts=None,
matcher=None):
"""Prefetches all the necessary file revisions for the given revs
Optionally runs repack in background
"""
with self._lock(
self.svfs,
"prefetchlock",
True,
None,
None,
_("prefetching in %s") % self.origroot,
):
self._prefetch(revs, base, pats, opts, matcher)
def _prefetch(self,
revs,
base=None,
pats=None,
opts=None,
matcher=None):
fallbackpath = self.fallbackpath
if fallbackpath:
# If we know a rev is on the server, we should fetch the server
# version of those files, since our local file versions might
# become obsolete if the local commits are stripped.
with progress.spinner(self.ui,
_("finding outgoing revisions")):
localrevs = self.revs("outgoing(%s)", fallbackpath)
if base is not None and base != nullrev:
serverbase = list(
self.revs("first(reverse(::%s) - %ld)", base,
localrevs))
if serverbase:
base = serverbase[0]
else:
localrevs = self
mfl = self.manifestlog
if base is not None:
mfdict = mfl[self[base].manifestnode()].read()
skip = set(mfdict.iteritems())
else:
skip = set()
# Copy the skip set to start large and avoid constant resizing,
# and since it's likely to be very similar to the prefetch set.
files = skip.copy()
serverfiles = skip.copy()
visited = set()
visited.add(nullid)
with progress.bar(self.ui, _("prefetching"),
total=len(revs)) as prog:
for rev in sorted(revs):
ctx = self[rev]
if pats:
m = scmutil.match(ctx, pats, opts)
if matcher is None:
matcher = self.maybesparsematch(rev)
mfnode = ctx.manifestnode()
mfctx = mfl[mfnode]
# Decompressing manifests is expensive.
# When possible, only read the deltas.
p1, p2 = mfctx.parents
if p1 in visited and p2 in visited:
mfdict = mfctx.readnew()
else:
mfdict = mfctx.read()
diff = mfdict.iteritems()
if pats:
diff = (pf for pf in diff if m(pf[0]))
if matcher:
diff = (pf for pf in diff if matcher(pf[0]))
if rev not in localrevs:
serverfiles.update(diff)
else:
files.update(diff)
visited.add(mfctx.node())
prog.value += 1
files.difference_update(skip)
serverfiles.difference_update(skip)
# Fetch files known to be on the server
if serverfiles:
results = [(path, hex(fnode)) for (path, fnode) in serverfiles]
self.fileservice.prefetch(results, force=True)
# Fetch files that may or may not be on the server
if files:
results = [(path, hex(fnode)) for (path, fnode) in files]
self.fileservice.prefetch(results)
repo.__class__ = shallowrepository
repo.shallowmatch = match.always(repo.root, "")
repo.fileservice = fileserverclient.fileserverclient(repo)
repo.numtransactioncommits = 0
repo.includepattern = repo.ui.configlist("remotefilelog", "includepattern",
None)
repo.excludepattern = repo.ui.configlist("remotefilelog", "excludepattern",
None)
if repo.includepattern or repo.excludepattern:
repo.shallowmatch = match.match(repo.root, "", None,
repo.includepattern,
repo.excludepattern)
def wraprepo(repo):
class shallowrepository(repo.__class__):
@util.propertycache
def name(self):
return self.ui.config("remotefilelog", "reponame", "unknown")
@util.propertycache
def fallbackpath(self):
path = self.ui.config(
"remotefilelog",
"fallbackpath",
# fallbackrepo is the old, deprecated name
self.ui.config("remotefilelog", "fallbackrepo",
self.ui.config("paths", "default")),
)
if not path:
raise error.Abort("no remotefilelog server "
"configured - is your .hg/hgrc trusted?")
return path
@util.propertycache
def fileslog(self):
return remotefilelog.remotefileslog(self)
def maybesparsematch(self, *revs, **kwargs):
"""
A wrapper that allows the remotefilelog to invoke sparsematch() if
this is a sparse repository, or returns None if this is not a
sparse repository.
"""
if util.safehasattr(self, "sparsematch"):
return self.sparsematch(*revs, **kwargs)
return None
def file(self, f):
if f[0] == "/":
f = f[1:]
if self.shallowmatch(f):
return remotefilelog.remotefilelog(self.svfs, f, self)
else:
return super(shallowrepository, self).file(f)
def filectx(self, path, changeid=None, fileid=None):
if self.shallowmatch(path):
return remotefilectx.remotefilectx(self, path, changeid,
fileid)
else:
return super(shallowrepository,
self).filectx(path, changeid, fileid)
def close(self):
result = super(shallowrepository, self).close()
if "fileslog" in self.__dict__:
self.fileslog.abortpending()
return result
def commitpending(self):
super(shallowrepository, self).commitpending()
self.numtransactioncommits += 1
# In some cases, we can have many transactions in the same repo, in
# which case each one will create a packfile, let's trigger a repack at
# this point to bring the number of packfiles down to a reasonable
# number.
if self.numtransactioncommits >= self.ui.configint(
"remotefilelog", "commitsperrepack"):
domaintenancerepack(self)
self.numtransactioncommits = 0
def commitctx(self, ctx, error=False):
"""Add a new revision to current repository.
Revision information is passed via the context argument.
"""
# some contexts already have manifest nodes, they don't need any
# prefetching (for example if we're just editing a commit message
# we can reuse manifest
if not ctx.manifestnode():
# prefetch files that will likely be compared
m1 = ctx.p1().manifest()
files = []
for f in ctx.modified() + ctx.added():
fparent1 = m1.get(f, nullid)
if fparent1 != nullid:
files.append((f, hex(fparent1)))
self.fileservice.prefetch(files)
return super(shallowrepository, self).commitctx(ctx, error=error)
def backgroundprefetch(self,
revs,
base=None,
repack=False,
pats=None,
opts=None):
"""Runs prefetch in background with optional repack"""
cmd = [util.hgexecutable(), "-R", self.origroot, "prefetch"]
if repack:
cmd.append("--repack")
if revs:
cmd += ["-r", revs]
if base:
cmd += ["-b", base]
util.spawndetached(cmd)
def prefetch(self, revs, base=None, matcher=None):
"""Prefetches all the necessary file revisions for the given revs
Optionally runs repack in background
"""
with self._lock(
self.svfs,
"prefetchlock",
True,
None,
None,
_("prefetching in %s") % self.origroot,
):
self._prefetch(revs, base, matcher)
def _prefetch(self, revs, base=None, matcher=None):
mfl = self.manifestlog
# Copy the skip set to start large and avoid constant resizing,
# and since it's likely to be very similar to the prefetch set.
files = set()
basemf = self[base or nullid].manifest()
with progress.bar(self.ui, _("prefetching"),
total=len(revs)) as prog:
for rev in sorted(revs):
ctx = self[rev]
if matcher is None:
matcher = self.maybesparsematch(rev)
mfctx = ctx.manifestctx()
mf = mfctx.read()
for path, (new, old) in mf.diff(basemf, matcher).items():
if new[0]:
files.add((path, new[0]))
prog.value += 1
if files:
results = [(path, hex(fnode)) for (path, fnode) in files]
self.fileservice.prefetch(results)
repo.__class__ = shallowrepository
repo.shallowmatch = match.always(repo.root, "")
repo.fileservice = fileserverclient.fileserverclient(repo)
repo.numtransactioncommits = 0
repo.includepattern = repo.ui.configlist("remotefilelog", "includepattern",
None)
repo.excludepattern = repo.ui.configlist("remotefilelog", "excludepattern",
None)
if repo.includepattern or repo.excludepattern:
repo.shallowmatch = match.match(repo.root, "", None,
repo.includepattern,
repo.excludepattern)
def testDiff(self):
a = cstore.treemanifest(FakeDataStore())
zflags = hashflags()
mflags = hashflags()
a.set("abc/z", *zflags)
a.set("xyz/m", *mflags)
alinknode = hashflags()[0]
b = cstore.treemanifest(FakeDataStore())
b.set("abc/z", *zflags)
b.set("xyz/m", *mflags)
blinknode = hashflags()[0]
# Diff matching trees
# - uncommitted trees
diff = a.diff(b)
self.assertEquals(diff, {})
# - committed trees
dstore = FakeDataStore()
hstore = FakeHistoryStore()
for name, node, text, p1text, p1, p2 in a.finalize():
dstore.add(name, node, nullid, text)
hstore.add(name, node, p1, p2, alinknode, "")
for name, node, text, p1text, p1, p2 in b.finalize():
dstore.add(name, node, nullid, text)
hstore.add(name, node, p1, p2, blinknode, "")
diff = a.diff(b)
self.assertEquals(diff, {})
b2 = b.copy()
# Diff with modifications
newfileflags = hashflags()
newzflags = hashflags()
b2.set("newfile", *newfileflags)
b2.set("abc/z", *newzflags)
# - uncommitted trees
diff = a.diff(b2)
self.assertEquals(diff, {
"newfile": ((None, ""), newfileflags),
"abc/z": (zflags, newzflags)
})
# - uncommitted trees with matcher
matcher = matchmod.match("/", "/", ["abc/*"])
diff = a.diff(b2, matcher=matcher)
self.assertEquals(diff, {"abc/z": (zflags, newzflags)})
matcher = matchmod.match("/", "/", ["newfile"])
diff = a.diff(b2, matcher=matcher)
self.assertEquals(diff, {"newfile": ((None, ""), newfileflags)})
# - committed trees
for name, node, text, p1text, p1, p2 in b2.finalize(a):
dstore.add(name, node, nullid, text)
hstore.add(name, node, p1, p2, blinknode, "")
diff = a.diff(b2)
self.assertEquals(diff, {
"newfile": ((None, ""), newfileflags),
"abc/z": (zflags, newzflags)
})
# Diff with clean
diff = a.diff(b2, clean=True)
self.assertEquals(
diff,
{
"newfile": ((None, ""), newfileflags),
"abc/z": (zflags, newzflags),
"xyz/m": None,
},
)
