def makechangegroup(orig, repo, outgoing, version, source, *args, **kwargs):
if not requirement in repo.requirements:
return orig(repo, outgoing, version, source, *args, **kwargs)
original = repo.shallowmatch
try:
# if serving, only send files the clients has patterns for
if source == "serve":
bundlecaps = kwargs.get("bundlecaps")
includepattern = None
excludepattern = None
for cap in bundlecaps or []:
if cap.startswith("includepattern="):
raw = cap[len("includepattern="):]
if raw:
includepattern = raw.split("\0")
elif cap.startswith("excludepattern="):
raw = cap[len("excludepattern="):]
if raw:
excludepattern = raw.split("\0")
if includepattern or excludepattern:
repo.shallowmatch = match.match(repo.root, "", None,
includepattern, excludepattern)
else:
repo.shallowmatch = match.always(repo.root, "")
return orig(repo, outgoing, version, source, *args, **kwargs)
finally:
repo.shallowmatch = original
def _walkstreamfiles(orig, repo):
if state.shallowremote:
# if we are shallow ourselves, stream our local commits
if shallowrepo.requirement in repo.requirements:
striplen = len(repo.store.path) + 1
readdir = repo.store.rawvfs.readdir
visit = [
os.path.join(repo.store.path, "packs"),
os.path.join(repo.store.path, "data"),
]
while visit:
p = visit.pop()
try:
dirents = readdir(p, stat=True)
except OSError as ex:
if ex.errno != errno.ENOENT:
raise
continue
for f, kind, st in dirents:
fp = p + "/" + f
if kind == stat.S_IFREG:
if not fp.endswith(".i") and not fp.endswith(".d"):
n = util.pconvert(fp[striplen:])
yield (store.decodedir(n), n, st.st_size)
if kind == stat.S_IFDIR:
visit.append(fp)
shallowtrees = repo.ui.configbool("remotefilelog", "shallowtrees", False)
if "treemanifest" in repo.requirements and not shallowtrees:
for (u, e, s) in repo.store.datafiles():
if u.startswith("meta/") and (u.endswith(".i") or u.endswith(".d")):
yield (u, e, s)
# Return .d and .i files that do not match the shallow pattern
match = state.match
if match and not match.always():
for (u, e, s) in repo.store.datafiles():
f = u[5:-2] # trim data/... and .i/.d
if not state.match(f):
yield (u, e, s)
for x in repo.store.topfiles():
if shallowtrees and x[0][:15] == "00manifesttree.":
continue
if state.noflatmf and x[0][:11] == "00manifest.":
continue
yield x
elif shallowrepo.requirement in repo.requirements:
# don't allow cloning from a shallow repo to a full repo
# since it would require fetching every version of every
# file in order to create the revlogs.
raise error.Abort(_("Cannot clone from a shallow repo " "to a full repo."))
else:
for x in orig(repo):
yield x
def _commit(orig, self, *args, **kwargs):
if _disabled[0]:
return orig(self, *args, **kwargs)
with self.wlock(), self.lock(), self.transaction("dirsynccommit"):
matcher = args[3] if len(args) >= 4 else kwargs.get("match")
matcher = matcher or matchmod.always(self.root, "")
mirroredfiles = _updateworkingcopy(self, matcher)
if mirroredfiles and not matcher.always():
origmatch = matcher.matchfn
def extramatches(path):
return path in mirroredfiles or origmatch(path)
matcher.matchfn = extramatches
matcher._files.extend(mirroredfiles)
matcher._fileset.update(mirroredfiles)
return orig(self, *args, **kwargs)
def diff(self, m2, matcher=None):
# Older mercurial clients used diff(m2, clean=False). If a caller failed
# to specify clean as a keyword arg, it might get passed as match here.
assert not isinstance(matcher,
bool), "match must inherit from basematcher"
self.load()
if isinstance(m2, overlaymanifest):
m2.load()
# below code copied from manifest.py:manifestdict.diff
diff = {}
try:
m2flagget = m2.flags
except AttributeError:
# Mercurial <= 3.3
m2flagget = m2._flags.get
if matcher is None:
matcher = matchmod.always("", "")
for fn, n1 in pycompat.iteritems(self):
if not matcher(fn):
continue
fl1 = self._flags.get(fn, "")
n2 = m2.get(fn, None)
fl2 = m2flagget(fn, "")
if n2 is None:
fl2 = ""
if n1 != n2 or fl1 != fl2:
diff[fn] = ((n1, fl1), (n2, fl2))
for fn, n2 in pycompat.iteritems(m2):
if fn not in self:
if not matcher(fn):
continue
fl2 = m2flagget(fn, "")
diff[fn] = ((None, ""), (n2, fl2))
return diff
def stream_out_shallow(repo, proto, other):
includepattern = None
excludepattern = None
raw = other.get("includepattern")
if raw:
includepattern = raw.split("\0")
raw = other.get("excludepattern")
if raw:
excludepattern = raw.split("\0")
oldshallow = state.shallowremote
oldmatch = state.match
oldnoflatmf = state.noflatmf
try:
state.shallowremote = True
state.match = match.always(repo.root, "")
state.noflatmf = other.get("noflatmanifest") == "True"
if includepattern or excludepattern:
state.match = match.match(
repo.root, "", None, includepattern, excludepattern
)
streamres = wireproto.stream(repo, proto)
# Force the first value to execute, so the file list is computed
# within the try/finally scope
first = next(streamres.gen)
second = next(streamres.gen)
def gen():
yield first
yield second
for value in streamres.gen:
yield value
return wireproto.streamres(gen())
finally:
state.shallowremote = oldshallow
state.match = oldmatch
state.noflatmf = oldnoflatmf
def generatefiles(self, changedfiles, linknodes, commonrevs, source):
if self._repo.ui.configbool("remotefilelog", "server"):
caps = self._bundlecaps or []
if requirement in caps:
# only send files that don't match the specified patterns
includepattern = None
excludepattern = None
for cap in self._bundlecaps or []:
if cap.startswith("includepattern="):
includepattern = cap[len("includepattern="):].split(
"\0")
elif cap.startswith("excludepattern="):
excludepattern = cap[len("excludepattern="):].split(
"\0")
m = match.always(self._repo.root, "")
if includepattern or excludepattern:
m = match.match(self._repo.root, "", None, includepattern,
excludepattern)
changedfiles = list([f for f in changedfiles if not m(f)])
if requirement in self._repo.requirements:
repo = self._repo
if isinstance(repo, bundlerepo.bundlerepository):
# If the bundle contains filelogs, we can't pull from it, since
# bundlerepo is heavily tied to revlogs. Instead require that
# the user use unbundle instead.
# Force load the filelog data.
bundlerepo.bundlerepository.file(repo, "foo")
if repo._cgfilespos:
raise error.Abort(
"cannot pull from full bundles",
hint="use `hg unbundle` instead",
)
return []
filestosend = self.shouldaddfilegroups(source)
if filestosend == NoFiles:
changedfiles = list(
[f for f in changedfiles if not repo.shallowmatch(f)])
else:
files = []
phasecache = repo._phasecache
cl = repo.changelog
# Prefetch the revisions being bundled
for i, fname in enumerate(sorted(changedfiles)):
filerevlog = repo.file(fname)
linkrevnodes = linknodes(filerevlog, fname)
# Normally we'd prune the linkrevnodes first,
# but that would perform the server fetches one by one.
for fnode, cnode in list(pycompat.iteritems(linkrevnodes)):
# Adjust linknodes so remote file revisions aren't sent
if filestosend == LocalFiles:
if phasecache.phase(
repo, cl.rev(cnode)
) == phases.public and repo.shallowmatch(fname):
del linkrevnodes[fnode]
else:
files.append((fname, hex(fnode)))
else:
files.append((fname, hex(fnode)))
repo.fileservice.prefetch(files)
# Prefetch the revisions that are going to be diffed against
prevfiles = []
for fname, fnode in files:
if repo.shallowmatch(fname):
fnode = bin(fnode)
filerevlog = repo.file(fname)
p1, p2, linknode, copyfrom = filerevlog.getnodeinfo(
fnode)
if p1 != nullid:
prevfiles.append((copyfrom or fname, hex(p1)))
repo.fileservice.prefetch(prevfiles)
return super(shallowcg1packer,
self).generatefiles(changedfiles, linknodes, commonrevs,
source)
def overridestatus(
orig,
self,
node1=".",
node2=None,
match=None,
ignored=False,
clean=False,
unknown=False,
):
listignored = ignored
listclean = clean
listunknown = unknown
def _cmpsets(l1, l2):
try:
if "FSMONITOR_LOG_FILE" in encoding.environ:
fn = encoding.environ["FSMONITOR_LOG_FILE"]
f = open(fn, "wb")
else:
fn = "fsmonitorfail.log"
f = self.opener(fn, "wb")
except (IOError, OSError):
self.ui.warn(_("warning: unable to write to %s\n") % fn)
return
try:
for i, (s1, s2) in enumerate(zip(l1, l2)):
if set(s1) != set(s2):
f.write("sets at position %d are unequal\n" % i)
f.write("watchman returned: %s\n" % s1)
f.write("stat returned: %s\n" % s2)
finally:
f.close()
if isinstance(node1, context.changectx):
ctx1 = node1
else:
ctx1 = self[node1]
if isinstance(node2, context.changectx):
ctx2 = node2
else:
ctx2 = self[node2]
working = ctx2.rev() is None
parentworking = working and ctx1 == self["."]
match = match or matchmod.always(self.root, self.getcwd())
# Maybe we can use this opportunity to update Watchman's state.
# Mercurial uses workingcommitctx and/or memctx to represent the part of
# the workingctx that is to be committed. So don't update the state in
# that case.
# HG_PENDING is set in the environment when the dirstate is being updated
# in the middle of a transaction; we must not update our state in that
# case, or we risk forgetting about changes in the working copy.
updatestate = (parentworking and match.always() and not isinstance(
ctx2,
(context.workingcommitctx, context.overlayworkingctx, context.memctx))
and "HG_PENDING" not in encoding.environ)
try:
if self._fsmonitorstate.walk_on_invalidate:
# Use a short timeout to query the current clock. If that
# takes too long then we assume that the service will be slow
# to answer our query.
# walk_on_invalidate indicates that we prefer to walk the
# tree ourselves because we can ignore portions that Watchman
# cannot and we tend to be faster in the warmer buffer cache
# cases.
self._watchmanclient.settimeout(0.1)
else:
# Give Watchman more time to potentially complete its walk
# and return the initial clock. In this mode we assume that
# the filesystem will be slower than parsing a potentially
# very large Watchman result set.
self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1)
startclock = self._watchmanclient.getcurrentclock()
except Exception as ex:
self._watchmanclient.clearconnection()
_handleunavailable(self.ui, self._fsmonitorstate, ex)
# boo, Watchman failed.
if self.ui.configbool("fsmonitor", "fallback-on-watchman-exception"):
return orig(node1, node2, match, listignored, listclean,
listunknown)
else:
raise ex
if updatestate:
# We need info about unknown files. This may make things slower the
# first time, but whatever.
stateunknown = True
else:
stateunknown = listunknown
if updatestate:
if "treestate" in self.requirements:
# No need to invalidate fsmonitor state.
# state.set needs to run before dirstate write, since it changes
# dirstate (treestate).
self.addpostdsstatus(poststatustreestate, afterdirstatewrite=False)
else:
# Invalidate fsmonitor.state if dirstate changes. This avoids the
# following issue:
# 1. pid 11 writes dirstate
# 2. pid 22 reads dirstate and inconsistent fsmonitor.state
# 3. pid 22 calculates a wrong state
# 4. pid 11 writes fsmonitor.state
# Because before 1,
# 0. pid 11 invalidates fsmonitor.state
# will happen.
#
# To avoid race conditions when reading without a lock, do things
# in this order:
# 1. Invalidate fsmonitor state
# 2. Write dirstate
# 3. Write fsmonitor state
psbefore = lambda *args, **kwds: self._fsmonitorstate.invalidate(
reason="dirstate_change")
self.addpostdsstatus(psbefore, afterdirstatewrite=False)
psafter = poststatus(startclock)
self.addpostdsstatus(psafter, afterdirstatewrite=True)
r = orig(node1, node2, match, listignored, listclean, stateunknown)
modified, added, removed, deleted, unknown, ignored, clean = r
if not listunknown:
unknown = []
# don't do paranoid checks if we're not going to query Watchman anyway
full = listclean or match.traversedir is not None
if self._fsmonitorstate.mode == "paranoid" and not full:
# run status again and fall back to the old walk this time
self.dirstate._fsmonitordisable = True
# shut the UI up
quiet = self.ui.quiet
self.ui.quiet = True
fout, ferr = self.ui.fout, self.ui.ferr
self.ui.fout = self.ui.ferr = open(os.devnull, "wb")
try:
rv2 = orig(node1, node2, match, listignored, listclean,
listunknown)
finally:
self.dirstate._fsmonitordisable = False
self.ui.quiet = quiet
self.ui.fout, self.ui.ferr = fout, ferr
# clean isn't tested since it's set to True above
_cmpsets([modified, added, removed, deleted, unknown, ignored, clean],
rv2)
modified, added, removed, deleted, unknown, ignored, clean = rv2
return scmutil.status(modified, added, removed, deleted, unknown, ignored,
clean)
def wraprepo(repo):
class shallowrepository(repo.__class__):
@util.propertycache
def name(self):
return self.ui.config("remotefilelog", "reponame", "")
@util.propertycache
def fallbackpath(self):
path = self.ui.config(
"remotefilelog",
"fallbackpath",
# fallbackrepo is the old, deprecated name
self.ui.config("remotefilelog", "fallbackrepo",
self.ui.config("paths", "default")),
)
if not path:
raise error.Abort("no remotefilelog server "
"configured - is your .hg/hgrc trusted?")
return path
@localrepo.unfilteredpropertycache
def fileslog(self):
return remotefilelog.remotefileslog(self)
def maybesparsematch(self, *revs, **kwargs):
"""
A wrapper that allows the remotefilelog to invoke sparsematch() if
this is a sparse repository, or returns None if this is not a
sparse repository.
"""
if util.safehasattr(self, "sparsematch"):
return self.sparsematch(*revs, **kwargs)
return None
def file(self, f):
if f[0] == "/":
f = f[1:]
if self.shallowmatch(f):
return remotefilelog.remotefilelog(self.svfs, f, self)
else:
return super(shallowrepository, self).file(f)
def filectx(self, path, changeid=None, fileid=None):
if self.shallowmatch(path):
return remotefilectx.remotefilectx(self, path, changeid,
fileid)
else:
return super(shallowrepository,
self).filectx(path, changeid, fileid)
@localrepo.unfilteredmethod
def close(self):
result = super(shallowrepository, self).close()
self.fileservice.close()
if "fileslog" in self.__dict__:
self.fileslog.abortpending()
return result
@localrepo.unfilteredmethod
def commitpending(self):
super(shallowrepository, self).commitpending()
self.numtransactioncommits += 1
# In some cases, we can have many transactions in the same repo, in
# which case each one will create a packfile, let's trigger a repack at
# this point to bring the number of packfiles down to a reasonable
# number.
if self.numtransactioncommits >= self.ui.configint(
"remotefilelog", "commitsperrepack"):
domaintenancerepack(self)
self.numtransactioncommits = 0
@localrepo.unfilteredmethod
def commitctx(self, ctx, error=False):
"""Add a new revision to current repository.
Revision information is passed via the context argument.
"""
# some contexts already have manifest nodes, they don't need any
# prefetching (for example if we're just editing a commit message
# we can reuse manifest
if not ctx.manifestnode():
# prefetch files that will likely be compared
m1 = ctx.p1().manifest()
files = []
for f in ctx.modified() + ctx.added():
fparent1 = m1.get(f, nullid)
if fparent1 != nullid:
files.append((f, hex(fparent1)))
self.fileservice.prefetch(files)
return super(shallowrepository, self).commitctx(ctx, error=error)
def backgroundprefetch(self,
revs,
base=None,
repack=False,
pats=None,
opts=None):
"""Runs prefetch in background with optional repack
"""
cmd = [util.hgexecutable(), "-R", self.origroot, "prefetch"]
if repack:
cmd.append("--repack")
if revs:
cmd += ["-r", revs]
if base:
cmd += ["-b", base]
cmd = " ".join(map(util.shellquote, cmd))
runshellcommand(cmd, encoding.environ)
def prefetch(self,
revs,
base=None,
pats=None,
opts=None,
matcher=None):
"""Prefetches all the necessary file revisions for the given revs
Optionally runs repack in background
"""
with self._lock(
self.svfs,
"prefetchlock",
True,
None,
None,
_("prefetching in %s") % self.origroot,
):
self._prefetch(revs, base, pats, opts, matcher)
def _prefetch(self,
revs,
base=None,
pats=None,
opts=None,
matcher=None):
fallbackpath = self.fallbackpath
if fallbackpath:
# If we know a rev is on the server, we should fetch the server
# version of those files, since our local file versions might
# become obsolete if the local commits are stripped.
with progress.spinner(self.ui,
_("finding outgoing revisions")):
localrevs = self.revs("outgoing(%s)", fallbackpath)
if base is not None and base != nullrev:
serverbase = list(
self.revs("first(reverse(::%s) - %ld)", base,
localrevs))
if serverbase:
base = serverbase[0]
else:
localrevs = self
mfl = self.manifestlog
if base is not None:
mfdict = mfl[self[base].manifestnode()].read()
skip = set(mfdict.iteritems())
else:
skip = set()
# Copy the skip set to start large and avoid constant resizing,
# and since it's likely to be very similar to the prefetch set.
files = skip.copy()
serverfiles = skip.copy()
visited = set()
visited.add(nullid)
with progress.bar(self.ui, _("prefetching"),
total=len(revs)) as prog:
for rev in sorted(revs):
ctx = self[rev]
if pats:
m = scmutil.match(ctx, pats, opts)
if matcher is None:
matcher = self.maybesparsematch(rev)
mfnode = ctx.manifestnode()
mfctx = mfl[mfnode]
# Decompressing manifests is expensive.
# When possible, only read the deltas.
p1, p2 = mfctx.parents
if p1 in visited and p2 in visited:
mfdict = mfctx.readnew()
else:
mfdict = mfctx.read()
diff = mfdict.iteritems()
if pats:
diff = (pf for pf in diff if m(pf[0]))
if matcher:
diff = (pf for pf in diff if matcher(pf[0]))
if rev not in localrevs:
serverfiles.update(diff)
else:
files.update(diff)
visited.add(mfctx.node())
prog.value += 1
files.difference_update(skip)
serverfiles.difference_update(skip)
# Fetch files known to be on the server
if serverfiles:
results = [(path, hex(fnode)) for (path, fnode) in serverfiles]
self.fileservice.prefetch(results, force=True)
# Fetch files that may or may not be on the server
if files:
results = [(path, hex(fnode)) for (path, fnode) in files]
self.fileservice.prefetch(results)
repo.__class__ = shallowrepository
repo.shallowmatch = match.always(repo.root, "")
repo.fileservice = fileserverclient.fileserverclient(repo)
repo.numtransactioncommits = 0
repo.includepattern = repo.ui.configlist("remotefilelog", "includepattern",
None)
repo.excludepattern = repo.ui.configlist("remotefilelog", "excludepattern",
None)
if repo.includepattern or repo.excludepattern:
repo.shallowmatch = match.match(repo.root, "", None,
repo.includepattern,
repo.excludepattern)
def wraprepo(repo):
class shallowrepository(repo.__class__):
@util.propertycache
def name(self):
return self.ui.config("remotefilelog", "reponame", "unknown")
@util.propertycache
def fallbackpath(self):
path = self.ui.config(
"remotefilelog",
"fallbackpath",
# fallbackrepo is the old, deprecated name
self.ui.config("remotefilelog", "fallbackrepo",
self.ui.config("paths", "default")),
)
if not path:
raise error.Abort("no remotefilelog server "
"configured - is your .hg/hgrc trusted?")
return path
@util.propertycache
def fileslog(self):
return remotefilelog.remotefileslog(self)
def maybesparsematch(self, *revs, **kwargs):
"""
A wrapper that allows the remotefilelog to invoke sparsematch() if
this is a sparse repository, or returns None if this is not a
sparse repository.
"""
if util.safehasattr(self, "sparsematch"):
return self.sparsematch(*revs, **kwargs)
return None
def file(self, f):
if f[0] == "/":
f = f[1:]
if self.shallowmatch(f):
return remotefilelog.remotefilelog(self.svfs, f, self)
else:
return super(shallowrepository, self).file(f)
def filectx(self, path, changeid=None, fileid=None):
if self.shallowmatch(path):
return remotefilectx.remotefilectx(self, path, changeid,
fileid)
else:
return super(shallowrepository,
self).filectx(path, changeid, fileid)
def close(self):
result = super(shallowrepository, self).close()
if "fileslog" in self.__dict__:
self.fileslog.abortpending()
return result
def commitpending(self):
super(shallowrepository, self).commitpending()
self.numtransactioncommits += 1
# In some cases, we can have many transactions in the same repo, in
# which case each one will create a packfile, let's trigger a repack at
# this point to bring the number of packfiles down to a reasonable
# number.
if self.numtransactioncommits >= self.ui.configint(
"remotefilelog", "commitsperrepack"):
domaintenancerepack(self)
self.numtransactioncommits = 0
def commitctx(self, ctx, error=False):
"""Add a new revision to current repository.
Revision information is passed via the context argument.
"""
# some contexts already have manifest nodes, they don't need any
# prefetching (for example if we're just editing a commit message
# we can reuse manifest
if not ctx.manifestnode():
# prefetch files that will likely be compared
m1 = ctx.p1().manifest()
files = []
for f in ctx.modified() + ctx.added():
fparent1 = m1.get(f, nullid)
if fparent1 != nullid:
files.append((f, hex(fparent1)))
self.fileservice.prefetch(files)
return super(shallowrepository, self).commitctx(ctx, error=error)
def backgroundprefetch(self,
revs,
base=None,
repack=False,
pats=None,
opts=None):
"""Runs prefetch in background with optional repack"""
cmd = [util.hgexecutable(), "-R", self.origroot, "prefetch"]
if repack:
cmd.append("--repack")
if revs:
cmd += ["-r", revs]
if base:
cmd += ["-b", base]
util.spawndetached(cmd)
def prefetch(self, revs, base=None, matcher=None):
"""Prefetches all the necessary file revisions for the given revs
Optionally runs repack in background
"""
with self._lock(
self.svfs,
"prefetchlock",
True,
None,
None,
_("prefetching in %s") % self.origroot,
):
self._prefetch(revs, base, matcher)
def _prefetch(self, revs, base=None, matcher=None):
mfl = self.manifestlog
# Copy the skip set to start large and avoid constant resizing,
# and since it's likely to be very similar to the prefetch set.
files = set()
basemf = self[base or nullid].manifest()
with progress.bar(self.ui, _("prefetching"),
total=len(revs)) as prog:
for rev in sorted(revs):
ctx = self[rev]
if matcher is None:
matcher = self.maybesparsematch(rev)
mfctx = ctx.manifestctx()
mf = mfctx.read()
for path, (new, old) in mf.diff(basemf, matcher).items():
if new[0]:
files.add((path, new[0]))
prog.value += 1
if files:
results = [(path, hex(fnode)) for (path, fnode) in files]
self.fileservice.prefetch(results)
repo.__class__ = shallowrepository
repo.shallowmatch = match.always(repo.root, "")
repo.fileservice = fileserverclient.fileserverclient(repo)
repo.numtransactioncommits = 0
repo.includepattern = repo.ui.configlist("remotefilelog", "includepattern",
None)
repo.excludepattern = repo.ui.configlist("remotefilelog", "excludepattern",
None)
if repo.includepattern or repo.excludepattern:
repo.shallowmatch = match.match(repo.root, "", None,
repo.includepattern,
repo.excludepattern)
