def makechangegroup(orig, repo, outgoing, version, source, *args, **kwargs): if not requirement in repo.requirements: return orig(repo, outgoing, version, source, *args, **kwargs) original = repo.shallowmatch try: # if serving, only send files the clients has patterns for if source == "serve": bundlecaps = kwargs.get("bundlecaps") includepattern = None excludepattern = None for cap in bundlecaps or []: if cap.startswith("includepattern="): raw = cap[len("includepattern="):] if raw: includepattern = raw.split("\0") elif cap.startswith("excludepattern="): raw = cap[len("excludepattern="):] if raw: excludepattern = raw.split("\0") if includepattern or excludepattern: repo.shallowmatch = match.match(repo.root, "", None, includepattern, excludepattern) else: repo.shallowmatch = match.always(repo.root, "") return orig(repo, outgoing, version, source, *args, **kwargs) finally: repo.shallowmatch = original
def _walkstreamfiles(orig, repo): if state.shallowremote: # if we are shallow ourselves, stream our local commits if shallowrepo.requirement in repo.requirements: striplen = len(repo.store.path) + 1 readdir = repo.store.rawvfs.readdir visit = [ os.path.join(repo.store.path, "packs"), os.path.join(repo.store.path, "data"), ] while visit: p = visit.pop() try: dirents = readdir(p, stat=True) except OSError as ex: if ex.errno != errno.ENOENT: raise continue for f, kind, st in dirents: fp = p + "/" + f if kind == stat.S_IFREG: if not fp.endswith(".i") and not fp.endswith(".d"): n = util.pconvert(fp[striplen:]) yield (store.decodedir(n), n, st.st_size) if kind == stat.S_IFDIR: visit.append(fp) shallowtrees = repo.ui.configbool("remotefilelog", "shallowtrees", False) if "treemanifest" in repo.requirements and not shallowtrees: for (u, e, s) in repo.store.datafiles(): if u.startswith("meta/") and (u.endswith(".i") or u.endswith(".d")): yield (u, e, s) # Return .d and .i files that do not match the shallow pattern match = state.match if match and not match.always(): for (u, e, s) in repo.store.datafiles(): f = u[5:-2] # trim data/... and .i/.d if not state.match(f): yield (u, e, s) for x in repo.store.topfiles(): if shallowtrees and x[0][:15] == "00manifesttree.": continue if state.noflatmf and x[0][:11] == "00manifest.": continue yield x elif shallowrepo.requirement in repo.requirements: # don't allow cloning from a shallow repo to a full repo # since it would require fetching every version of every # file in order to create the revlogs. raise error.Abort(_("Cannot clone from a shallow repo " "to a full repo.")) else: for x in orig(repo): yield x
def _commit(orig, self, *args, **kwargs): if _disabled[0]: return orig(self, *args, **kwargs) with self.wlock(), self.lock(), self.transaction("dirsynccommit"): matcher = args[3] if len(args) >= 4 else kwargs.get("match") matcher = matcher or matchmod.always(self.root, "") mirroredfiles = _updateworkingcopy(self, matcher) if mirroredfiles and not matcher.always(): origmatch = matcher.matchfn def extramatches(path): return path in mirroredfiles or origmatch(path) matcher.matchfn = extramatches matcher._files.extend(mirroredfiles) matcher._fileset.update(mirroredfiles) return orig(self, *args, **kwargs)
def diff(self, m2, matcher=None): # Older mercurial clients used diff(m2, clean=False). If a caller failed # to specify clean as a keyword arg, it might get passed as match here. assert not isinstance(matcher, bool), "match must inherit from basematcher" self.load() if isinstance(m2, overlaymanifest): m2.load() # below code copied from manifest.py:manifestdict.diff diff = {} try: m2flagget = m2.flags except AttributeError: # Mercurial <= 3.3 m2flagget = m2._flags.get if matcher is None: matcher = matchmod.always("", "") for fn, n1 in pycompat.iteritems(self): if not matcher(fn): continue fl1 = self._flags.get(fn, "") n2 = m2.get(fn, None) fl2 = m2flagget(fn, "") if n2 is None: fl2 = "" if n1 != n2 or fl1 != fl2: diff[fn] = ((n1, fl1), (n2, fl2)) for fn, n2 in pycompat.iteritems(m2): if fn not in self: if not matcher(fn): continue fl2 = m2flagget(fn, "") diff[fn] = ((None, ""), (n2, fl2)) return diff
def stream_out_shallow(repo, proto, other): includepattern = None excludepattern = None raw = other.get("includepattern") if raw: includepattern = raw.split("\0") raw = other.get("excludepattern") if raw: excludepattern = raw.split("\0") oldshallow = state.shallowremote oldmatch = state.match oldnoflatmf = state.noflatmf try: state.shallowremote = True state.match = match.always(repo.root, "") state.noflatmf = other.get("noflatmanifest") == "True" if includepattern or excludepattern: state.match = match.match( repo.root, "", None, includepattern, excludepattern ) streamres = wireproto.stream(repo, proto) # Force the first value to execute, so the file list is computed # within the try/finally scope first = next(streamres.gen) second = next(streamres.gen) def gen(): yield first yield second for value in streamres.gen: yield value return wireproto.streamres(gen()) finally: state.shallowremote = oldshallow state.match = oldmatch state.noflatmf = oldnoflatmf
def generatefiles(self, changedfiles, linknodes, commonrevs, source): if self._repo.ui.configbool("remotefilelog", "server"): caps = self._bundlecaps or [] if requirement in caps: # only send files that don't match the specified patterns includepattern = None excludepattern = None for cap in self._bundlecaps or []: if cap.startswith("includepattern="): includepattern = cap[len("includepattern="):].split( "\0") elif cap.startswith("excludepattern="): excludepattern = cap[len("excludepattern="):].split( "\0") m = match.always(self._repo.root, "") if includepattern or excludepattern: m = match.match(self._repo.root, "", None, includepattern, excludepattern) changedfiles = list([f for f in changedfiles if not m(f)]) if requirement in self._repo.requirements: repo = self._repo if isinstance(repo, bundlerepo.bundlerepository): # If the bundle contains filelogs, we can't pull from it, since # bundlerepo is heavily tied to revlogs. Instead require that # the user use unbundle instead. # Force load the filelog data. bundlerepo.bundlerepository.file(repo, "foo") if repo._cgfilespos: raise error.Abort( "cannot pull from full bundles", hint="use `hg unbundle` instead", ) return [] filestosend = self.shouldaddfilegroups(source) if filestosend == NoFiles: changedfiles = list( [f for f in changedfiles if not repo.shallowmatch(f)]) else: files = [] phasecache = repo._phasecache cl = repo.changelog # Prefetch the revisions being bundled for i, fname in enumerate(sorted(changedfiles)): filerevlog = repo.file(fname) linkrevnodes = linknodes(filerevlog, fname) # Normally we'd prune the linkrevnodes first, # but that would perform the server fetches one by one. for fnode, cnode in list(pycompat.iteritems(linkrevnodes)): # Adjust linknodes so remote file revisions aren't sent if filestosend == LocalFiles: if phasecache.phase( repo, cl.rev(cnode) ) == phases.public and repo.shallowmatch(fname): del linkrevnodes[fnode] else: files.append((fname, hex(fnode))) else: files.append((fname, hex(fnode))) repo.fileservice.prefetch(files) # Prefetch the revisions that are going to be diffed against prevfiles = [] for fname, fnode in files: if repo.shallowmatch(fname): fnode = bin(fnode) filerevlog = repo.file(fname) p1, p2, linknode, copyfrom = filerevlog.getnodeinfo( fnode) if p1 != nullid: prevfiles.append((copyfrom or fname, hex(p1))) repo.fileservice.prefetch(prevfiles) return super(shallowcg1packer, self).generatefiles(changedfiles, linknodes, commonrevs, source)
def overridestatus( orig, self, node1=".", node2=None, match=None, ignored=False, clean=False, unknown=False, ): listignored = ignored listclean = clean listunknown = unknown def _cmpsets(l1, l2): try: if "FSMONITOR_LOG_FILE" in encoding.environ: fn = encoding.environ["FSMONITOR_LOG_FILE"] f = open(fn, "wb") else: fn = "fsmonitorfail.log" f = self.opener(fn, "wb") except (IOError, OSError): self.ui.warn(_("warning: unable to write to %s\n") % fn) return try: for i, (s1, s2) in enumerate(zip(l1, l2)): if set(s1) != set(s2): f.write("sets at position %d are unequal\n" % i) f.write("watchman returned: %s\n" % s1) f.write("stat returned: %s\n" % s2) finally: f.close() if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = self[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = self[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self["."] match = match or matchmod.always(self.root, self.getcwd()) # Maybe we can use this opportunity to update Watchman's state. # Mercurial uses workingcommitctx and/or memctx to represent the part of # the workingctx that is to be committed. So don't update the state in # that case. # HG_PENDING is set in the environment when the dirstate is being updated # in the middle of a transaction; we must not update our state in that # case, or we risk forgetting about changes in the working copy. updatestate = (parentworking and match.always() and not isinstance( ctx2, (context.workingcommitctx, context.overlayworkingctx, context.memctx)) and "HG_PENDING" not in encoding.environ) try: if self._fsmonitorstate.walk_on_invalidate: # Use a short timeout to query the current clock. If that # takes too long then we assume that the service will be slow # to answer our query. # walk_on_invalidate indicates that we prefer to walk the # tree ourselves because we can ignore portions that Watchman # cannot and we tend to be faster in the warmer buffer cache # cases. self._watchmanclient.settimeout(0.1) else: # Give Watchman more time to potentially complete its walk # and return the initial clock. In this mode we assume that # the filesystem will be slower than parsing a potentially # very large Watchman result set. self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1) startclock = self._watchmanclient.getcurrentclock() except Exception as ex: self._watchmanclient.clearconnection() _handleunavailable(self.ui, self._fsmonitorstate, ex) # boo, Watchman failed. if self.ui.configbool("fsmonitor", "fallback-on-watchman-exception"): return orig(node1, node2, match, listignored, listclean, listunknown) else: raise ex if updatestate: # We need info about unknown files. This may make things slower the # first time, but whatever. stateunknown = True else: stateunknown = listunknown if updatestate: if "treestate" in self.requirements: # No need to invalidate fsmonitor state. # state.set needs to run before dirstate write, since it changes # dirstate (treestate). self.addpostdsstatus(poststatustreestate, afterdirstatewrite=False) else: # Invalidate fsmonitor.state if dirstate changes. This avoids the # following issue: # 1. pid 11 writes dirstate # 2. pid 22 reads dirstate and inconsistent fsmonitor.state # 3. pid 22 calculates a wrong state # 4. pid 11 writes fsmonitor.state # Because before 1, # 0. pid 11 invalidates fsmonitor.state # will happen. # # To avoid race conditions when reading without a lock, do things # in this order: # 1. Invalidate fsmonitor state # 2. Write dirstate # 3. Write fsmonitor state psbefore = lambda *args, **kwds: self._fsmonitorstate.invalidate( reason="dirstate_change") self.addpostdsstatus(psbefore, afterdirstatewrite=False) psafter = poststatus(startclock) self.addpostdsstatus(psafter, afterdirstatewrite=True) r = orig(node1, node2, match, listignored, listclean, stateunknown) modified, added, removed, deleted, unknown, ignored, clean = r if not listunknown: unknown = [] # don't do paranoid checks if we're not going to query Watchman anyway full = listclean or match.traversedir is not None if self._fsmonitorstate.mode == "paranoid" and not full: # run status again and fall back to the old walk this time self.dirstate._fsmonitordisable = True # shut the UI up quiet = self.ui.quiet self.ui.quiet = True fout, ferr = self.ui.fout, self.ui.ferr self.ui.fout = self.ui.ferr = open(os.devnull, "wb") try: rv2 = orig(node1, node2, match, listignored, listclean, listunknown) finally: self.dirstate._fsmonitordisable = False self.ui.quiet = quiet self.ui.fout, self.ui.ferr = fout, ferr # clean isn't tested since it's set to True above _cmpsets([modified, added, removed, deleted, unknown, ignored, clean], rv2) modified, added, removed, deleted, unknown, ignored, clean = rv2 return scmutil.status(modified, added, removed, deleted, unknown, ignored, clean)
def wraprepo(repo): class shallowrepository(repo.__class__): @util.propertycache def name(self): return self.ui.config("remotefilelog", "reponame", "") @util.propertycache def fallbackpath(self): path = self.ui.config( "remotefilelog", "fallbackpath", # fallbackrepo is the old, deprecated name self.ui.config("remotefilelog", "fallbackrepo", self.ui.config("paths", "default")), ) if not path: raise error.Abort("no remotefilelog server " "configured - is your .hg/hgrc trusted?") return path @localrepo.unfilteredpropertycache def fileslog(self): return remotefilelog.remotefileslog(self) def maybesparsematch(self, *revs, **kwargs): """ A wrapper that allows the remotefilelog to invoke sparsematch() if this is a sparse repository, or returns None if this is not a sparse repository. """ if util.safehasattr(self, "sparsematch"): return self.sparsematch(*revs, **kwargs) return None def file(self, f): if f[0] == "/": f = f[1:] if self.shallowmatch(f): return remotefilelog.remotefilelog(self.svfs, f, self) else: return super(shallowrepository, self).file(f) def filectx(self, path, changeid=None, fileid=None): if self.shallowmatch(path): return remotefilectx.remotefilectx(self, path, changeid, fileid) else: return super(shallowrepository, self).filectx(path, changeid, fileid) @localrepo.unfilteredmethod def close(self): result = super(shallowrepository, self).close() self.fileservice.close() if "fileslog" in self.__dict__: self.fileslog.abortpending() return result @localrepo.unfilteredmethod def commitpending(self): super(shallowrepository, self).commitpending() self.numtransactioncommits += 1 # In some cases, we can have many transactions in the same repo, in # which case each one will create a packfile, let's trigger a repack at # this point to bring the number of packfiles down to a reasonable # number. if self.numtransactioncommits >= self.ui.configint( "remotefilelog", "commitsperrepack"): domaintenancerepack(self) self.numtransactioncommits = 0 @localrepo.unfilteredmethod def commitctx(self, ctx, error=False): """Add a new revision to current repository. Revision information is passed via the context argument. """ # some contexts already have manifest nodes, they don't need any # prefetching (for example if we're just editing a commit message # we can reuse manifest if not ctx.manifestnode(): # prefetch files that will likely be compared m1 = ctx.p1().manifest() files = [] for f in ctx.modified() + ctx.added(): fparent1 = m1.get(f, nullid) if fparent1 != nullid: files.append((f, hex(fparent1))) self.fileservice.prefetch(files) return super(shallowrepository, self).commitctx(ctx, error=error) def backgroundprefetch(self, revs, base=None, repack=False, pats=None, opts=None): """Runs prefetch in background with optional repack """ cmd = [util.hgexecutable(), "-R", self.origroot, "prefetch"] if repack: cmd.append("--repack") if revs: cmd += ["-r", revs] if base: cmd += ["-b", base] cmd = " ".join(map(util.shellquote, cmd)) runshellcommand(cmd, encoding.environ) def prefetch(self, revs, base=None, pats=None, opts=None, matcher=None): """Prefetches all the necessary file revisions for the given revs Optionally runs repack in background """ with self._lock( self.svfs, "prefetchlock", True, None, None, _("prefetching in %s") % self.origroot, ): self._prefetch(revs, base, pats, opts, matcher) def _prefetch(self, revs, base=None, pats=None, opts=None, matcher=None): fallbackpath = self.fallbackpath if fallbackpath: # If we know a rev is on the server, we should fetch the server # version of those files, since our local file versions might # become obsolete if the local commits are stripped. with progress.spinner(self.ui, _("finding outgoing revisions")): localrevs = self.revs("outgoing(%s)", fallbackpath) if base is not None and base != nullrev: serverbase = list( self.revs("first(reverse(::%s) - %ld)", base, localrevs)) if serverbase: base = serverbase[0] else: localrevs = self mfl = self.manifestlog if base is not None: mfdict = mfl[self[base].manifestnode()].read() skip = set(mfdict.iteritems()) else: skip = set() # Copy the skip set to start large and avoid constant resizing, # and since it's likely to be very similar to the prefetch set. files = skip.copy() serverfiles = skip.copy() visited = set() visited.add(nullid) with progress.bar(self.ui, _("prefetching"), total=len(revs)) as prog: for rev in sorted(revs): ctx = self[rev] if pats: m = scmutil.match(ctx, pats, opts) if matcher is None: matcher = self.maybesparsematch(rev) mfnode = ctx.manifestnode() mfctx = mfl[mfnode] # Decompressing manifests is expensive. # When possible, only read the deltas. p1, p2 = mfctx.parents if p1 in visited and p2 in visited: mfdict = mfctx.readnew() else: mfdict = mfctx.read() diff = mfdict.iteritems() if pats: diff = (pf for pf in diff if m(pf[0])) if matcher: diff = (pf for pf in diff if matcher(pf[0])) if rev not in localrevs: serverfiles.update(diff) else: files.update(diff) visited.add(mfctx.node()) prog.value += 1 files.difference_update(skip) serverfiles.difference_update(skip) # Fetch files known to be on the server if serverfiles: results = [(path, hex(fnode)) for (path, fnode) in serverfiles] self.fileservice.prefetch(results, force=True) # Fetch files that may or may not be on the server if files: results = [(path, hex(fnode)) for (path, fnode) in files] self.fileservice.prefetch(results) repo.__class__ = shallowrepository repo.shallowmatch = match.always(repo.root, "") repo.fileservice = fileserverclient.fileserverclient(repo) repo.numtransactioncommits = 0 repo.includepattern = repo.ui.configlist("remotefilelog", "includepattern", None) repo.excludepattern = repo.ui.configlist("remotefilelog", "excludepattern", None) if repo.includepattern or repo.excludepattern: repo.shallowmatch = match.match(repo.root, "", None, repo.includepattern, repo.excludepattern)
def wraprepo(repo): class shallowrepository(repo.__class__): @util.propertycache def name(self): return self.ui.config("remotefilelog", "reponame", "unknown") @util.propertycache def fallbackpath(self): path = self.ui.config( "remotefilelog", "fallbackpath", # fallbackrepo is the old, deprecated name self.ui.config("remotefilelog", "fallbackrepo", self.ui.config("paths", "default")), ) if not path: raise error.Abort("no remotefilelog server " "configured - is your .hg/hgrc trusted?") return path @util.propertycache def fileslog(self): return remotefilelog.remotefileslog(self) def maybesparsematch(self, *revs, **kwargs): """ A wrapper that allows the remotefilelog to invoke sparsematch() if this is a sparse repository, or returns None if this is not a sparse repository. """ if util.safehasattr(self, "sparsematch"): return self.sparsematch(*revs, **kwargs) return None def file(self, f): if f[0] == "/": f = f[1:] if self.shallowmatch(f): return remotefilelog.remotefilelog(self.svfs, f, self) else: return super(shallowrepository, self).file(f) def filectx(self, path, changeid=None, fileid=None): if self.shallowmatch(path): return remotefilectx.remotefilectx(self, path, changeid, fileid) else: return super(shallowrepository, self).filectx(path, changeid, fileid) def close(self): result = super(shallowrepository, self).close() if "fileslog" in self.__dict__: self.fileslog.abortpending() return result def commitpending(self): super(shallowrepository, self).commitpending() self.numtransactioncommits += 1 # In some cases, we can have many transactions in the same repo, in # which case each one will create a packfile, let's trigger a repack at # this point to bring the number of packfiles down to a reasonable # number. if self.numtransactioncommits >= self.ui.configint( "remotefilelog", "commitsperrepack"): domaintenancerepack(self) self.numtransactioncommits = 0 def commitctx(self, ctx, error=False): """Add a new revision to current repository. Revision information is passed via the context argument. """ # some contexts already have manifest nodes, they don't need any # prefetching (for example if we're just editing a commit message # we can reuse manifest if not ctx.manifestnode(): # prefetch files that will likely be compared m1 = ctx.p1().manifest() files = [] for f in ctx.modified() + ctx.added(): fparent1 = m1.get(f, nullid) if fparent1 != nullid: files.append((f, hex(fparent1))) self.fileservice.prefetch(files) return super(shallowrepository, self).commitctx(ctx, error=error) def backgroundprefetch(self, revs, base=None, repack=False, pats=None, opts=None): """Runs prefetch in background with optional repack""" cmd = [util.hgexecutable(), "-R", self.origroot, "prefetch"] if repack: cmd.append("--repack") if revs: cmd += ["-r", revs] if base: cmd += ["-b", base] util.spawndetached(cmd) def prefetch(self, revs, base=None, matcher=None): """Prefetches all the necessary file revisions for the given revs Optionally runs repack in background """ with self._lock( self.svfs, "prefetchlock", True, None, None, _("prefetching in %s") % self.origroot, ): self._prefetch(revs, base, matcher) def _prefetch(self, revs, base=None, matcher=None): mfl = self.manifestlog # Copy the skip set to start large and avoid constant resizing, # and since it's likely to be very similar to the prefetch set. files = set() basemf = self[base or nullid].manifest() with progress.bar(self.ui, _("prefetching"), total=len(revs)) as prog: for rev in sorted(revs): ctx = self[rev] if matcher is None: matcher = self.maybesparsematch(rev) mfctx = ctx.manifestctx() mf = mfctx.read() for path, (new, old) in mf.diff(basemf, matcher).items(): if new[0]: files.add((path, new[0])) prog.value += 1 if files: results = [(path, hex(fnode)) for (path, fnode) in files] self.fileservice.prefetch(results) repo.__class__ = shallowrepository repo.shallowmatch = match.always(repo.root, "") repo.fileservice = fileserverclient.fileserverclient(repo) repo.numtransactioncommits = 0 repo.includepattern = repo.ui.configlist("remotefilelog", "includepattern", None) repo.excludepattern = repo.ui.configlist("remotefilelog", "excludepattern", None) if repo.includepattern or repo.excludepattern: repo.shallowmatch = match.match(repo.root, "", None, repo.includepattern, repo.excludepattern)