def logsyncop( repo, op, version, oldheads, newheads, oldbm, newbm, oldrbm, newrbm, oldsnapshots, newsnapshots, ): oldheadsset = set(oldheads) newheadsset = set(newheads) oldbmset = set(oldbm) newbmset = set(newbm) oldrbmset = set(oldrbm) newrbmset = set(newrbm) oldsnapset = set(oldsnapshots) newsnapset = set(newsnapshots) addedheads = blackbox.shortlist( [h for h in newheads if h not in oldheadsset]) removedheads = blackbox.shortlist( [h for h in oldheads if h not in newheadsset]) addedbm = blackbox.shortlist([h for h in newbm if h not in oldbmset]) removedbm = blackbox.shortlist([h for h in oldbm if h not in newbmset]) addedrbm = blackbox.shortlist([h for h in newrbm if h not in oldrbmset]) removedrbm = blackbox.shortlist([h for h in oldrbm if h not in newrbmset]) addedsnaps = blackbox.shortlist( [h for h in newsnapshots if h not in oldsnapset]) removedsnaps = blackbox.shortlist( [h for h in oldsnapshots if h not in newsnapset]) blackbox.log({ "commit_cloud_sync": { "op": op, "version": version, "added_heads": addedheads, "removed_heads": removedheads, "added_bookmarks": addedbm, "removed_bookmarks": removedbm, "added_remote_bookmarks": addedrbm, "removed_remote_bookmarks": removedrbm, "added_snapshots": addedsnaps, "removed_snapshots": removedsnaps, } }) util.info("commit-cloud-sync", op=op, version=version)
def _walk(self, match, event): """Replacement for filesystem._walk, hooking into Watchman. Whenever listignored is False and the Watchman client is available, use Watchman combined with saved state to possibly return only a subset of files.""" state = self._fsmonitorstate clock, ignorehash, notefiles = state.get() if not clock: if state.walk_on_invalidate: raise fsmonitorfallback("no clock") # Initial NULL clock value, see # https://facebook.github.io/watchman/docs/clockspec.html clock = "c:0:0" notefiles = [] ignore = self.dirstate._ignore # experimental config: experimental.fsmonitor.skipignore if not self._ui.configbool("experimental", "fsmonitor.skipignore"): if ignorehash and _hashignore( ignore) != ignorehash and clock != "c:0:0": # ignore list changed -- can't rely on Watchman state any more if state.walk_on_invalidate: raise fsmonitorfallback("ignore rules changed") notefiles = [] clock = "c:0:0" matchfn = match.matchfn matchalways = match.always() dmap = self.dirstate._map if util.safehasattr(dmap, "_map"): # for better performance, directly access the inner dirstate map if the # standard dirstate implementation is in use. dmap = dmap._map if "treestate" in self._repo.requirements: # treestate has a fast path to filter out ignored directories. ignorevisitdir = self.dirstate._ignore.visitdir def dirfilter(path): result = ignorevisitdir(path.rstrip("/")) return result == "all" nonnormalset = self.dirstate._map.nonnormalsetfiltered(dirfilter) else: nonnormalset = self.dirstate._map.nonnormalset event["old_clock"] = clock event["old_files"] = blackbox.shortlist(nonnormalset) copymap = self.dirstate._map.copymap getkind = stat.S_IFMT dirkind = stat.S_IFDIR regkind = stat.S_IFREG lnkkind = stat.S_IFLNK join = self.dirstate._join normcase = util.normcase fresh_instance = False exact = False if match.isexact(): # match.exact exact = True if not exact and self.dirstate._checkcase: # note that even though we could receive directory entries, we're only # interested in checking if a file with the same name exists. So only # normalize files if possible. normalize = self.dirstate._normalizefile else: normalize = None # step 2: query Watchman try: # Use the user-configured timeout for the query. # Add a little slack over the top of the user query to allow for # overheads while transferring the data self._watchmanclient.settimeout(state.timeout + 0.1) result = self._watchmanclient.command( "query", { "fields": ["mode", "mtime", "size", "exists", "name"], "since": clock, "expression": [ "not", [ "anyof", ["dirname", ".hg"], ["name", ".hg", "wholename"] ], ], "sync_timeout": int(state.timeout * 1000), "empty_on_fresh_instance": state.walk_on_invalidate, }, ) except Exception as ex: event["is_error"] = True _handleunavailable(self._ui, state, ex) self._watchmanclient.clearconnection() # XXX: Legacy scuba logging. Remove this once the source of truth # is moved to the Rust Event. self._ui.log("fsmonitor_status", fsmonitor_status="exception") if self._ui.configbool("fsmonitor", "fallback-on-watchman-exception"): raise fsmonitorfallback("exception during run") else: raise ex else: # We need to propagate the last observed clock up so that we # can use it for our next query event["new_clock"] = result["clock"] event["is_fresh"] = result["is_fresh_instance"] state.setlastclock(result["clock"]) state.setlastisfresh(result["is_fresh_instance"]) if result["is_fresh_instance"]: if not self._ui.plain() and self._ui.configbool( "fsmonitor", "warn-fresh-instance"): oldpid = _watchmanpid(event["old_clock"]) newpid = _watchmanpid(event["new_clock"]) if oldpid is not None and newpid is not None: self._ui.warn( _("warning: watchman has recently restarted (old pid %s, new pid %s) - operation will be slower than usual\n" ) % (oldpid, newpid)) elif oldpid is None and newpid is not None: self._ui.warn( _("warning: watchman has recently started (pid %s) - operation will be slower than usual\n" ) % (newpid, )) else: self._ui.warn( _("warning: watchman failed to catch up with file change events and requires a full scan - operation will be slower than usual\n" )) if state.walk_on_invalidate: state.invalidate(reason="fresh_instance") raise fsmonitorfallback("fresh instance") fresh_instance = True # Ignore any prior noteable files from the state info notefiles = [] else: count = len(result["files"]) state.setwatchmanchangedfilecount(count) event["new_files"] = blackbox.shortlist( (e["name"] for e in result["files"]), count) # XXX: Legacy scuba logging. Remove this once the source of truth # is moved to the Rust Event. if event["is_fresh"]: self._ui.log("fsmonitor_status", fsmonitor_status="fresh") else: self._ui.log("fsmonitor_status", fsmonitor_status="normal") results = {} # for file paths which require normalization and we encounter a case # collision, we store our own foldmap if normalize: foldmap = dict((normcase(k), k) for k in results) switch_slashes = pycompat.ossep == "\\" # The order of the results is, strictly speaking, undefined. # For case changes on a case insensitive filesystem we may receive # two entries, one with exists=True and another with exists=False. # The exists=True entries in the same response should be interpreted # as being happens-after the exists=False entries due to the way that # Watchman tracks files. We use this property to reconcile deletes # for name case changes. ignorelist = [] ignorelistappend = ignorelist.append for entry in result["files"]: fname = entry["name"] if _fixencoding: fname = _watchmantofsencoding(fname) if switch_slashes: fname = fname.replace("\\", "/") if normalize: normed = normcase(fname) fname = normalize(fname, True, True) foldmap[normed] = fname fmode = entry["mode"] fexists = entry["exists"] kind = getkind(fmode) if not fexists: # if marked as deleted and we don't already have a change # record, mark it as deleted. If we already have an entry # for fname then it was either part of walkexplicit or was # an earlier result that was a case change if (fname not in results and fname in dmap and (matchalways or matchfn(fname))): results[fname] = None elif kind == dirkind: if fname in dmap and (matchalways or matchfn(fname)): results[fname] = None elif kind == regkind or kind == lnkkind: if fname in dmap: if matchalways or matchfn(fname): results[fname] = entry else: ignored = ignore(fname) if ignored: ignorelistappend(fname) if (matchalways or matchfn(fname)) and not ignored: results[fname] = entry elif fname in dmap and (matchalways or matchfn(fname)): results[fname] = None elif fname in match.files(): match.bad(fname, filesystem.badtype(kind)) # step 3: query notable files we don't already know about # XXX try not to iterate over the entire dmap if normalize: # any notable files that have changed case will already be handled # above, so just check membership in the foldmap notefiles = set((normalize(f, True, True) for f in notefiles if normcase(f) not in foldmap)) visit = set((f for f in notefiles if ( f not in results and matchfn(f) and (f in dmap or not ignore(f))))) if not fresh_instance: if matchalways: visit.update(f for f in nonnormalset if f not in results) visit.update(f for f in copymap if f not in results) else: visit.update(f for f in nonnormalset if f not in results and matchfn(f)) visit.update(f for f in copymap if f not in results and matchfn(f)) else: if matchalways: visit.update(f for f in dmap if f not in results) visit.update(f for f in copymap if f not in results) else: visit.update(f for f in dmap if f not in results and matchfn(f)) visit.update(f for f in copymap if f not in results and matchfn(f)) # audit returns False for paths with one of its parent directories being a # symlink. audit = pathutil.pathauditor(self.dirstate._root, cached=True).check auditpass = [f for f in visit if audit(f)] auditpass.sort() auditfail = visit.difference(auditpass) droplist = [] droplistappend = droplist.append for f in auditfail: # For auditfail paths, they should be treated as not existed in working # copy. filestate = dmap.get(f, ("?", ))[0] if filestate in ("?", ): # do not exist in working parents, remove them from treestate and # avoid walking through them. droplistappend(f) results.pop(f, None) else: # tracked, mark as deleted results[f] = None nf = iter(auditpass).next for st in util.statfiles([join(f) for f in auditpass]): f = nf() if (st and not ignore(f)) or f in dmap: results[f] = st elif not st: # '?' (untracked) file was deleted from the filesystem - remove it # from treestate. # # We can only update the dirstate (and treestate) while holding the # wlock. That happens inside poststatus.__call__ -> state.set. So # buffer what files to "drop" so state.set can clean them up. entry = dmap.get(f, None) if entry and entry[0] == "?": droplistappend(f) # The droplist and ignorelist need to match setlastclock() state.setdroplist(droplist) state.setignorelist(ignorelist) results.pop(".hg", None) return results.iteritems()