def _peersetup(ui, peer):
if peer.capable("clienttelemetry"):
logargs = clienttelemetryvaluesfromconfig(ui)
logargs.update(
{name: f(ui)
for name, f in _clienttelemetryfuncs.items()})
logargs.update(_clienttelemetrydata)
response = decodeutf8(peer._call("clienttelemetry", **logargs))
responseitems = response.split()
peername = responseitems[0] if responseitems else ""
peer._realhostname = peername
peerinfo = {}
for index in range(1, len(responseitems) - 1, 2):
peerinfo[responseitems[index]] = responseitems[index + 1]
peer._peerinfo = peerinfo
blackbox.log(
{"clienttelemetry": {
"peername": peername,
"peerinfo": peerinfo
}})
util.info("client-telemetry", peername=peername, **peerinfo)
ann = ui.configbool("clienttelemetry", "announceremotehostname", None)
if ann is None:
ann = not ui.plain() and ui._isatty(ui.ferr)
if ann and not ui.quiet:
ui.write_err(_("connected to %s\n") % response)
perftrace.tracevalue("Server", peername)
for item, value in peerinfo.items():
perftrace.tracevalue(f"Server {item}", value)
def _lfsprefetch(self, fileids):
if not _lfsmod or not util.safehasattr(self.repo.svfs,
"lfslocalblobstore"):
return
if not _lfsmod.wrapper.candownload(self.repo):
return
pointers = []
filenames = {}
store = self.repo.svfs.lfslocalblobstore
for file, id in fileids:
node = bin(id)
rlog = self.repo.file(file)
if rlog.flags(node) & revlog.REVIDX_EXTSTORED:
text = rlog.revision(node, raw=True)
p = _lfsmod.pointer.deserialize(text)
oid = p.oid()
if not store.has(oid):
pointers.append(p)
filenames[oid] = file
if len(pointers) > 0:
perftrace.tracevalue("Missing", len(pointers))
self.repo.svfs.lfsremoteblobstore.readbatch(pointers,
store,
objectnames=filenames)
assert all(store.has(p.oid()) for p in pointers)
def _peersetup(ui, peer):
if peer.capable("clienttelemetry"):
logargs = {name: f(ui) for name, f in _clienttelemetryfuncs.items()}
logargs.update(_clienttelemetrydata)
peername = decodeutf8(peer._call("clienttelemetry", **logargs))
peer._realhostname = peername
blackbox.log({"clienttelemetry": {"peername": peername}})
ann = ui.configbool("clienttelemetry", "announceremotehostname", None)
if ann is None:
ann = not ui.plain() and ui._isatty(ui.ferr)
if ann and not ui.quiet:
ui.warn(_("connected to %s\n") % peername)
perftrace.tracevalue("Server", peername)
def prefetch(self, datastore, historystore, fileids):
total = len(fileids)
perftrace.tracevalue("Files requested", len(fileids))
try:
rcvd = None
if self.repo.ui.configbool("remotefilelog", "retryprefetch"):
retries = 0
for backoff in [1, 5, 10, 20]:
try:
rcvd = self.getpack(datastore, historystore, fileids)
break
except (error.BadResponseError, error.NetworkError):
missingids = set()
missingids.update(datastore.getmissing(fileids))
missingids.update(historystore.getmissing(fileids))
fileids = list(missingids)
self.ui.warn(
_(
"Network connection dropped while fetching data, retrying after %d seconds\n"
)
% backoff
)
time.sleep(backoff)
retries += 1
continue
if retries > 0:
perftrace.tracevalue("Retries", retries)
if rcvd is None:
rcvd = self.getpack(datastore, historystore, fileids)
self.ui.log(
"remotefilefetchlog",
"Success(pack)\n" if (rcvd == total) else "Fail(pack)\n",
fetched_files=rcvd,
total_to_fetch=total,
)
except Exception:
self.ui.log(
"remotefilefetchlog",
"Fail(pack)\n",
fetched_files=total - len(fileids),
total_to_fetch=total,
)
raise
def _linkrev(self):
if self._filenode == nullid:
return nullrev
p1, p2, linknode, copyfrom = self.getnodeinfo()
rev = self._repo.changelog.nodemap.get(linknode)
if rev is not None:
return rev
# Search all commits for the appropriate linkrev (slow, but uncommon)
repo = self._repo
path = self._path
fileid = self._filenode
cl = repo.changelog
mfl = repo.manifestlog
with repo.ui.timesection("scanlinkrev"), repo.ui.configoverride({
("treemanifest", "fetchdepth"):
1
}), perftrace.trace("Scanning for Linkrev"), progress.bar(
repo.ui,
_("scanning for linkrev of %s") % path) as prog:
perftrace.tracevalue("Path", path)
allrevs = repo.revs("_all()")
allrevs.sort(reverse=True)
for i, rev in enumerate(allrevs):
prog.value = i
node = cl.node(rev)
data = cl.read(
node) # get changeset data (we avoid object creation)
if path in data[3]: # checking the 'files' field.
# The file has been touched, check if the hash is what we're
# looking for.
if fileid == mfl[data[0]].read().get(path):
perftrace.tracevalue("Distance", i)
return rev
# Couldn't find the linkrev. This should generally not happen, and will
# likely cause a crash.
return None
def prefetch(self,
fileids,
force=False,
fetchdata=True,
fetchhistory=True):
"""downloads the given file versions to the cache
"""
repo = self.repo
idstocheck = set()
for file, id in fileids:
# hack
# - we don't use .hgtags
# - workingctx produces ids with length 42,
# which we skip since they aren't in any cache
if file == ".hgtags" or len(
id) == 42 or not repo.shallowmatch(file):
continue
idstocheck.add((file, bin(id)))
batchlfsdownloads = self.ui.configbool("remotefilelog",
"_batchlfsdownloads", True)
dolfsprefetch = self.ui.configbool("remotefilelog", "dolfsprefetch",
True)
idstocheck = list(idstocheck)
if repo.fileslog._ruststore:
if not force:
contentstore = repo.fileslog.contentstore
metadatastore = repo.fileslog.metadatastore
else:
contentstore, metadatastore = repo.fileslog.makesharedonlyruststore(
repo)
if fetchdata:
contentstore.prefetch(idstocheck)
if fetchhistory:
metadatastore.prefetch(idstocheck)
if batchlfsdownloads and dolfsprefetch:
self._lfsprefetch(fileids)
if force:
# Yay, since the shared-only stores and the regular ones aren't
# shared, we need to commit data to force the stores to be
# rebuilt. Forced prefetch are very rare and thus it is most
# likely OK to do this.
contentstore = None
metadatastore = None
repo.commitpending()
return
datastore = self.datastore
historystore = self.historystore
if force:
datastore = unioncontentstore(*repo.fileslog.shareddatastores)
historystore = unionmetadatastore(
*repo.fileslog.sharedhistorystores)
perftrace.tracevalue("Keys", len(idstocheck))
missingids = set()
if fetchdata:
missingids.update(datastore.getmissing(idstocheck))
perftrace.tracevalue("Missing Data", len(missingids))
if fetchhistory:
missinghistory = historystore.getmissing(idstocheck)
missingids.update(missinghistory)
perftrace.tracevalue("Missing History", len(missinghistory))
# partition missing nodes into nullid and not-nullid so we can
# warn about this filtering potentially shadowing bugs.
nullids = len([None for unused, id in missingids if id == nullid])
if nullids:
missingids = [(f, id) for f, id in missingids if id != nullid]
repo.ui.develwarn(
("remotefilelog not fetching %d null revs"
" - this is likely hiding bugs" % nullids),
config="remotefilelog-ext",
)
if missingids:
global fetches, fetched, fetchcost
fetches += 1
missingids = [(file, hex(id)) for file, id in missingids]
fetched += len(missingids)
start = time.time()
with self.ui.timesection("fetchingfiles"):
self.request(missingids, fetchdata, fetchhistory)
fetchcost += time.time() - start
if not batchlfsdownloads and dolfsprefetch:
self._lfsprefetch(fileids)
if batchlfsdownloads and dolfsprefetch:
self._lfsprefetch(fileids)
def requestpacks(self, fileids, fetchdata, fetchhistory):
self.remotecache.reconnect()
perftrace.traceflag("packs")
cache = self.remotecache
fileslog = self.repo.fileslog
total = len(fileids)
totalfetches = 0
if fetchdata:
totalfetches += total
if fetchhistory:
totalfetches += total
with progress.bar(self.ui,
_("fetching from memcache"),
total=totalfetches) as prog:
# generate `get` keys and make data request
getkeys = [file + "\0" + node for file, node in fileids]
if fetchdata:
cache.getdatapack(getkeys)
if fetchhistory:
cache.gethistorypack(getkeys)
# receive both data and history
misses = []
try:
allmisses = set()
if fetchdata:
allmisses.update(cache.receive(prog))
fileslog.contentstore.markforrefresh()
if fetchhistory:
allmisses.update(cache.receive(prog))
fileslog.metadatastore.markforrefresh()
misses = [key.split("\0") for key in allmisses]
perftrace.tracevalue("Memcache Misses", len(misses))
except CacheConnectionError:
misses = fileids
self.ui.warn(
_("warning: cache connection closed early - " +
"falling back to server\n"))
global fetchmisses
missedfiles = len(misses)
fetchmisses += missedfiles
fromcache = total - missedfiles
self.ui.log(
"remotefilelog",
"remote cache hit rate is %r of %r\n",
fromcache,
total,
hit=fromcache,
total=total,
)
oldumask = os.umask(0o002)
try:
# receive cache misses from master
if missedfiles > 0:
self._fetchpackfiles(misses, fetchdata, fetchhistory)
finally:
os.umask(oldumask)
def _adjustlinknode(self, path, filelog, fnode, srcrev, inclusive=False):
"""return the first ancestor of <srcrev> introducing <fnode>
If the linkrev of the file revision does not point to an ancestor of
srcrev, we'll walk down the ancestors until we find one introducing
this file revision.
:repo: a localrepository object (used to access changelog and manifest)
:path: the file path
:fnode: the nodeid of the file revision
:filelog: the filelog of this path
:srcrev: the changeset revision we search ancestors from
:inclusive: if true, the src revision will also be checked
Note: This is based on adjustlinkrev in core, but it's quite different.
adjustlinkrev depends on the fact that the linkrev is the bottom most
node, and uses that as a stopping point for the ancestor traversal. We
can't do that here because the linknode is not guaranteed to be the
bottom most one.
In our code here, we actually know what a bunch of potential ancestor
linknodes are, so instead of stopping the cheap-ancestor-traversal when
we get to a linkrev, we stop when we see any of the known linknodes.
"""
repo = self._repo
cl = repo.changelog
mfl = repo.manifestlog
linknode = self.getnodeinfo()[2]
if srcrev is None:
# wctx case, used by workingfilectx during mergecopy
revs = [p.rev() for p in self._repo[None].parents()]
inclusive = True # we skipped the real (revless) source
else:
revs = [srcrev]
if self._verifylinknode(revs, linknode):
return linknode
commonlogkwargs = {
"revs": " ".join([hex(cl.node(rev)) for rev in revs]),
"fnode": hex(fnode),
"filepath": path,
"user": shallowutil.getusername(repo.ui),
"reponame": shallowutil.getreponame(repo.ui),
}
repo.ui.log("linkrevfixup", "adjusting linknode", **commonlogkwargs)
# Adjustlinknodes accesses the file node in the manifest for a variety
# of manifests. Let's prevent us from downloading large numbers of trees
# by temporarily limiting the fetch depth to 1.
with repo.ui.timesection("adjustlinknode"), repo.ui.configoverride({
("treemanifest", "fetchdepth"):
1
}), perftrace.trace("Adjust Linknode"), progress.bar(
repo.ui,
_("adjusting linknode for %s") % self._path) as prog:
perftrace.tracevalue("Path", self._path)
perftrace.tracevalue("Source Nodes",
[hex(cl.node(rev)) for rev in revs])
pc = repo._phasecache
seenpublic = False
iterancs = repo.revs("reverse(::%ld)",
revs).prefetch("text").iterctx()
for i, ctx in enumerate(iterancs):
ancrev = ctx.rev()
if ancrev == srcrev and not inclusive:
continue
prog.value = i
# First, check locally-available history.
lnode = self._nodefromancrev(ancrev, cl, mfl, path, fnode)
if lnode is not None:
return lnode
# adjusting linknode can be super-slow. To mitigate the issue
# we use two heuristics: calling fastlog and forcing remotefilelog
# prefetch
if not seenpublic and pc.phase(repo, ancrev) == phases.public:
# If the commit is public and fastlog is enabled for this repo
# then we can try to fetch the right linknode via fastlog.
if repo.ui.configbool("fastlog", "enabled"):
lnode = self._linknodeviafastlog(
repo, path, ancrev, fnode, cl, mfl,
commonlogkwargs)
if lnode:
return lnode
# If fastlog is not enabled and/or failed, let's try
# prefetching
lnode = self._forceprefetch(repo, path, fnode, revs,
commonlogkwargs)
if lnode:
return lnode
seenpublic = True
return linknode
