def _buildjournalentry(self, namespace, name, oldhashes, newhashes): if not isinstance(oldhashes, list): oldhashes = [oldhashes] if not isinstance(newhashes, list): newhashes = [newhashes] return journalentry( util.makedate(), self.user, self.command, namespace, name, oldhashes, newhashes, )
def listcmd(ui, repo, pats, opts): """subcommand that displays the list of shelves""" pats = set(pats) width = 80 if not ui.plain(): width = ui.termwidth() namelabel = "shelve.newest" ui.pager("shelve") for mtime, name in listshelves(repo): sname = util.split(name)[1] if pats and sname not in pats: continue ui.write(sname, label=namelabel) namelabel = "shelve.name" if ui.quiet: ui.write("\n") continue ui.write(" " * (16 - len(sname))) used = 16 age = "(%s)" % templatefilters.age(util.makedate(mtime), abbrev=True) ui.write(age, label="shelve.age") ui.write(" " * (12 - len(age))) used += 12 with open(name + "." + patchextension, "rb") as fp: while True: line = fp.readline() if not line: break if not line.startswith(b"#"): desc = pycompat.decodeutf8(line.rstrip()) if ui.formatted: desc = util.ellipsis(desc, width - used) ui.write(desc) break ui.write("\n") if not (opts["patch"] or opts["stat"]): continue difflines = fp.readlines() if opts["patch"]: for chunk, label in patch.difflabel(iter, difflines): ui.writebytes(chunk, label=label) if opts["stat"]: for chunk, label in patch.diffstatui(difflines, width=width): ui.write(chunk, label=label)
def backupdisable(ui, repo, **opts): """temporarily disable automatic backup or sync Disables automatic background backup or sync for the specified duration. """ if not background.autobackupenabled(repo): ui.write(_("background backup was already disabled\n"), notice=_("note")) try: duration = int(opts.get("hours", 1)) * 60 * 60 except ValueError: raise error.Abort( _( "error: argument 'hours': invalid int value: '{value}'\n".format( value=opts.get("hours") ) ) ) timestamp = int(time.time()) + duration background.disableautobackup(repo, timestamp) ui.write( _("background backup is now disabled until %s\n") % util.datestr(util.makedate(timestamp)), component="commitcloud", ) try: with backuplock.trylock(repo): pass except error.LockHeld as e: if e.lockinfo.isrunning(): ui.warn( _( "'@prog@ cloud disable' does not affect running backup processes\n" "(kill the background process - pid %s on %s - gracefully if needed)\n" ) % (e.lockinfo.uniqueid, e.lockinfo.namespace), notice=_("warning"), ) return 0
def _applycloudchanges(repo, remotepath, lastsyncstate, cloudrefs, maxage, state, tr): # Pull all the new heads and any bookmark hashes we don't have. We need to # filter cloudrefs before pull as pull doesn't check if a rev is present # locally. unfi = repo newheads = [head for head in cloudrefs.heads if head not in unfi] if maxage is not None and maxage >= 0: mindate = time.time() - maxage * 86400 omittedheads = [ head for head in newheads if head in cloudrefs.headdates and cloudrefs.headdates[head] < mindate ] if omittedheads: repo.ui.status(_("omitting heads that are older than %d days:\n") % maxage) for head in omittedheads: headdatestr = util.datestr(util.makedate(cloudrefs.headdates[head])) repo.ui.status(_(" %s from %s\n") % (head[:12], headdatestr)) newheads = [head for head in newheads if head not in omittedheads] else: omittedheads = [] omittedbookmarks = [] omittedremotebookmarks = [] newvisibleheads = None if visibility.tracking(repo): localheads = _getheads(repo) localheadsset = set(localheads) cloudheads = [head for head in cloudrefs.heads if head not in omittedheads] cloudheadsset = set(cloudheads) if localheadsset != cloudheadsset: oldvisibleheads = [ head for head in lastsyncstate.heads if head not in lastsyncstate.omittedheads ] newvisibleheads = util.removeduplicates( oldvisibleheads + cloudheads + localheads ) toremove = { head for head in oldvisibleheads if head not in localheadsset or head not in cloudheadsset } newvisibleheads = [head for head in newvisibleheads if head not in toremove] remotebookmarknewnodes = set() remotebookmarkupdates = {} if _isremotebookmarkssyncenabled(repo.ui): (remotebookmarkupdates, remotebookmarknewnodes) = _processremotebookmarks( repo, cloudrefs.remotebookmarks, lastsyncstate ) try: snapshot = extensions.find("snapshot") except KeyError: snapshot = None addedsnapshots = [] removedsnapshots = [] newsnapshots = lastsyncstate.snapshots else: addedsnapshots = [ s for s in cloudrefs.snapshots if s not in lastsyncstate.snapshots ] removedsnapshots = [ s for s in lastsyncstate.snapshots if s not in cloudrefs.snapshots ] newsnapshots = cloudrefs.snapshots newheads += addedsnapshots if remotebookmarknewnodes or newheads: # Partition the heads into groups we can pull together. headgroups = _partitionheads( list(remotebookmarknewnodes) + newheads, cloudrefs.headdates ) _pullheadgroups(repo, remotepath, headgroups) omittedbookmarks.extend( _mergebookmarks(repo, tr, cloudrefs.bookmarks, lastsyncstate) ) newremotebookmarks = {} if _isremotebookmarkssyncenabled(repo.ui): newremotebookmarks, omittedremotebookmarks = _updateremotebookmarks( repo, tr, remotebookmarkupdates ) if snapshot: with repo.lock(), repo.transaction("sync-snapshots") as tr: repo.snapshotlist.update( tr, addnodes=addedsnapshots, removenodes=removedsnapshots ) _mergeobsmarkers(repo, tr, cloudrefs.obsmarkers) if newvisibleheads is not None: visibility.setvisibleheads(repo, [nodemod.bin(n) for n in newvisibleheads]) # Obsmarker sharing is unreliable. Some of the commits that should now # be visible might be hidden still, and some commits that should be # hidden might still be visible. Create local obsmarkers to resolve # this. if obsolete.isenabled(repo, obsolete.createmarkersopt) and not repo.ui.configbool( "mutation", "proxy-obsstore" ): unfi = repo # Commits that are only visible in the cloud are commits that are # ancestors of the cloud heads but are hidden locally. cloudvisibleonly = list( unfi.set( "not public() & ::%ls & hidden()", [head for head in cloudrefs.heads if head not in omittedheads], ) ) # Commits that are only hidden in the cloud are commits that are # ancestors of the previous cloud heads that are not ancestors of the # current cloud heads, but have not been hidden or obsoleted locally. cloudhiddenonly = list( unfi.set( "(not public() & ::%ls) - (not public() & ::%ls) - hidden() - obsolete()", [ head for head in lastsyncstate.heads if head not in lastsyncstate.omittedheads ], [head for head in cloudrefs.heads if head not in omittedheads], ) ) if cloudvisibleonly or cloudhiddenonly: msg = _( "detected obsmarker inconsistency (fixing by obsoleting [%s] and reviving [%s])\n" ) % ( ", ".join([nodemod.short(ctx.node()) for ctx in cloudhiddenonly]), ", ".join([nodemod.short(ctx.node()) for ctx in cloudvisibleonly]), ) repo.ui.log("commitcloud_sync", msg) repo.ui.warn(msg) repo._commitcloudskippendingobsmarkers = True with repo.lock(): obsolete.createmarkers(repo, [(ctx, ()) for ctx in cloudhiddenonly]) obsolete.revive(cloudvisibleonly) repo._commitcloudskippendingobsmarkers = False # We have now synced the repo to the cloud version. Store this. logsyncop( repo, "from_cloud", cloudrefs.version, lastsyncstate.heads, cloudrefs.heads, lastsyncstate.bookmarks, cloudrefs.bookmarks, lastsyncstate.remotebookmarks, newremotebookmarks, lastsyncstate.snapshots, newsnapshots, ) lastsyncstate.update( tr, newversion=cloudrefs.version, newheads=cloudrefs.heads, newbookmarks=cloudrefs.bookmarks, newremotebookmarks=newremotebookmarks, newmaxage=maxage, newomittedheads=omittedheads, newomittedbookmarks=omittedbookmarks, newomittedremotebookmarks=omittedremotebookmarks, newsnapshots=newsnapshots, ) # Also update backup state. These new heads are already backed up, # otherwise the server wouldn't have told us about them. state.update([nodemod.bin(head) for head in newheads], tr)
def commitfunc(**kwargs): if not repo.ui.configbool("tweakdefaults", "histeditkeepdate"): kwargs["date"] = util.makedate(time.time()) origcommitfunc(**kwargs)
def currentdate(): return "%d %d" % util.makedate(time.time())
def _applycloudchanges(repo, remotepath, lastsyncstate, cloudrefs, maxage, state, tr): # Pull all the new heads and any bookmark hashes we don't have. We need to # filter cloudrefs before pull as pull doesn't check if a rev is present # locally. newheads = [ nodemod.hex(n) for n in repo.changelog.filternodes( [nodemod.bin(h) for h in cloudrefs.heads], inverse=True) ] assert newheads == newheads if maxage is not None and maxage >= 0: mindate = time.time() - maxage * 86400 omittedheads = [ head for head in newheads if head in cloudrefs.headdates and cloudrefs.headdates[head] < mindate ] if omittedheads: omittedheadslen = len(omittedheads) repo.ui.status( _n( "omitting %d head that is older than %d days:\n", "omitting %d heads that are older than %d days:\n", omittedheadslen, ) % (omittedheadslen, maxage)) counter = 0 for head in reversed(omittedheads): if counter == _maxomittedheadsoutput: remaining = len(omittedheads) - counter repo.ui.status( _n(" and %d older head\n", " and %d older heads\n", remaining) % remaining) break headdatestr = util.datestr( util.makedate(cloudrefs.headdates[head])) repo.ui.status(_(" %s from %s\n") % (head[:12], headdatestr)) counter = counter + 1 omittedheads = set(omittedheads) newheads = [head for head in newheads if head not in omittedheads] else: omittedheads = set() omittedbookmarks = [] omittedremotebookmarks = [] newvisibleheads = None if visibility.tracking(repo): localheads = _getheads(repo) localheadsset = set(localheads) cloudheads = [ head for head in cloudrefs.heads if head not in omittedheads ] cloudheadsset = set(cloudheads) if localheadsset != cloudheadsset: oldvisibleheads = [ head for head in lastsyncstate.heads if head not in lastsyncstate.omittedheads ] newvisibleheads = util.removeduplicates(oldvisibleheads + cloudheads + localheads) toremove = { head for head in oldvisibleheads if head not in localheadsset or head not in cloudheadsset } newvisibleheads = [ head for head in newvisibleheads if head not in toremove ] remotebookmarknewnodes = set() remotebookmarkupdates = {} if _isremotebookmarkssyncenabled(repo.ui): (remotebookmarkupdates, remotebookmarknewnodes) = _processremotebookmarks( repo, cloudrefs.remotebookmarks, lastsyncstate) if remotebookmarknewnodes or newheads: # Partition the heads into groups we can pull together. headgroups = _partitionheads(repo.ui, list(remotebookmarknewnodes) + newheads, cloudrefs.headdates) _pullheadgroups(repo, remotepath, headgroups) omittedbookmarks.extend( _mergebookmarks(repo, tr, cloudrefs.bookmarks, lastsyncstate, omittedheads, maxage)) newremotebookmarks = {} if _isremotebookmarkssyncenabled(repo.ui): omittedremotebookmarks = _updateremotebookmarks( repo, tr, remotebookmarkupdates) newremotebookmarks = cloudrefs.remotebookmarks if newvisibleheads is not None: visibility.setvisibleheads(repo, [nodemod.bin(n) for n in newvisibleheads]) # We have now synced the repo to the cloud version. Store this. logsyncop( repo, "from_cloud", cloudrefs.version, lastsyncstate.heads, cloudrefs.heads, lastsyncstate.bookmarks, cloudrefs.bookmarks, lastsyncstate.remotebookmarks, newremotebookmarks, ) lastsyncstate.update( tr, newversion=cloudrefs.version, newheads=cloudrefs.heads, newbookmarks=cloudrefs.bookmarks, newremotebookmarks=newremotebookmarks, newmaxage=maxage, newomittedheads=list(omittedheads), newomittedbookmarks=omittedbookmarks, newomittedremotebookmarks=omittedremotebookmarks, ) # Also update backup state. These new heads are already backed up, # otherwise the server wouldn't have told us about them. state.update([nodemod.bin(head) for head in newheads], tr)
def synthesize(ui, repo, descpath, **opts): """synthesize commits based on a model of an existing repository The model must have been generated by :hg:`analyze`. Commits will be generated randomly according to the probabilities described in the model. If --initfiles is set, the repository will be seeded with the given number files following the modeled repository's directory structure. When synthesizing new content, commit descriptions, and user names, words will be chosen randomly from a dictionary that is presumed to contain one word per line. Use --dict to specify the path to an alternate dictionary to use. """ try: fp = hg.openpath(ui, descpath) except Exception as err: raise error.Abort("%s: %s" % (descpath, err[0].strerror)) desc = json.load(fp) fp.close() def cdf(l): if not l: return [], [] vals, probs = zip(*sorted(l, key=lambda x: x[1], reverse=True)) t = float(sum(probs, 0)) s, cdfs = 0, [] for v in probs: s += v cdfs.append(s / t) return vals, cdfs lineschanged = cdf(desc["lineschanged"]) fileschanged = cdf(desc["fileschanged"]) filesadded = cdf(desc["filesadded"]) dirsadded = cdf(desc["dirsadded"]) filesremoved = cdf(desc["filesremoved"]) linelengths = cdf(desc["linelengths"]) parents = cdf(desc["parents"]) p1distance = cdf(desc["p1distance"]) p2distance = cdf(desc["p2distance"]) interarrival = cdf(desc["interarrival"]) linesinfilesadded = cdf(desc["linesinfilesadded"]) tzoffset = cdf(desc["tzoffset"]) dictfile = opts.get("dict") or "/usr/share/dict/words" try: fp = open(dictfile, "rU") except IOError as err: raise error.Abort("%s: %s" % (dictfile, err.strerror)) words = fp.read().splitlines() fp.close() initdirs = {} if desc["initdirs"]: for k, v in desc["initdirs"]: initdirs[k.encode("utf-8").replace(".hg", "_hg")] = v initdirs = renamedirs(initdirs, words) initdirscdf = cdf(initdirs) def pick(cdf): return cdf[0][bisect.bisect_left(cdf[1], random.random())] def pickpath(): return os.path.join(pick(initdirscdf), random.choice(words)) def makeline(minimum=0): total = max(minimum, pick(linelengths)) c, l = 0, [] while c < total: w = random.choice(words) c += len(w) + 1 l.append(w) return " ".join(l) wlock = repo.wlock() lock = repo.lock() nevertouch = {".hgsub", ".hgignore", ".hgtags"} progress = ui.progress _synthesizing = _("synthesizing") _files = _("initial files") _changesets = _("changesets") # Synthesize a single initial revision adding files to the repo according # to the modeled directory structure. initcount = int(opts["initfiles"]) if initcount and initdirs: pctx = repo[None].parents()[0] dirs = set(pctx.dirs()) files = {} def validpath(path): # Don't pick filenames which are already directory names. if path in dirs: return False # Don't pick directories which were used as file names. while path: if path in files: return False path = os.path.dirname(path) return True for i in xrange(0, initcount): ui.progress(_synthesizing, i, unit=_files, total=initcount) path = pickpath() while not validpath(path): path = pickpath() data = "%s contents\n" % path files[path] = data dir = os.path.dirname(path) while dir and dir not in dirs: dirs.add(dir) dir = os.path.dirname(dir) def filectxfn(repo, memctx, path): return context.memfilectx(repo, memctx, path, files[path]) ui.progress(_synthesizing, None) message = "synthesized wide repo with %d files" % (len(files), ) mc = context.memctx( repo, [pctx.node(), nullid], message, pycompat.iterkeys(files), filectxfn, ui.username(), "%d %d" % util.makedate(), ) initnode = mc.commit() if ui.debugflag: hexfn = hex else: hexfn = short ui.status( _("added commit %s with %d files\n") % (hexfn(initnode), len(files))) # Synthesize incremental revisions to the repository, adding repo depth. count = int(opts["count"]) heads = set(map(repo.changelog.rev, repo.heads())) for i in xrange(count): progress(_synthesizing, i, unit=_changesets, total=count) node = repo.changelog.node revs = len(repo) def pickhead(heads, distance): if heads: lheads = sorted(heads) rev = revs - min(pick(distance), revs) if rev < lheads[-1]: rev = lheads[bisect.bisect_left(lheads, rev)] else: rev = lheads[-1] return rev, node(rev) return nullrev, nullid r1 = revs - min(pick(p1distance), revs) p1 = node(r1) # the number of heads will grow without bound if we use a pure # model, so artificially constrain their proliferation toomanyheads = len(heads) > random.randint(1, 20) if p2distance[0] and (pick(parents) == 2 or toomanyheads): r2, p2 = pickhead(heads.difference([r1]), p2distance) else: r2, p2 = nullrev, nullid pl = [p1, p2] pctx = repo[r1] mf = pctx.manifest() mfk = mf.keys() changes = {} if mfk: for __ in xrange(pick(fileschanged)): for __ in xrange(10): fctx = pctx.filectx(random.choice(mfk)) path = fctx.path() if not (path in nevertouch or fctx.isbinary() or "l" in fctx.flags()): break lines = fctx.data().splitlines() add, remove = pick(lineschanged) for __ in xrange(remove): if not lines: break del lines[random.randrange(0, len(lines))] for __ in xrange(add): lines.insert(random.randint(0, len(lines)), makeline()) path = fctx.path() changes[path] = "\n".join(lines) + "\n" for __ in xrange(pick(filesremoved)): path = random.choice(mfk) for __ in xrange(10): path = random.choice(mfk) if path not in changes: break if filesadded: dirs = list(pctx.dirs()) dirs.insert(0, "") for __ in xrange(pick(filesadded)): pathstr = "" while pathstr in dirs: path = [random.choice(dirs)] if pick(dirsadded): path.append(random.choice(words)) path.append(random.choice(words)) pathstr = "/".join(filter(None, path)) data = ("\n".join(makeline() for __ in xrange(pick(linesinfilesadded))) + "\n") changes[pathstr] = data def filectxfn(repo, memctx, path): if path not in changes: return None return context.memfilectx(repo, memctx, path, changes[path]) if not changes: continue if revs: date = repo["tip"].date()[0] + pick(interarrival) else: date = time.time() - (86400 * count) # dates in mercurial must be positive, fit in 32-bit signed integers. date = min(0x7FFFFFFF, max(0, date)) user = random.choice(words) + "@" + random.choice(words) mc = context.memctx( repo, pl, makeline(minimum=2), sorted(changes), filectxfn, user, "%d %d" % (date, pick(tzoffset)), ) newnode = mc.commit() heads.add(repo.changelog.rev(newnode)) heads.discard(r1) heads.discard(r2) lock.release() wlock.release()
def summary(repo): ui = repo.ui # commitcloud config should eventually replace the infinitepushbackup one if not ui.configbool("infinitepushbackup", "enablestatus") or not ui.configbool( "commitcloud", "enablestatus"): return # Output backup status if enablestatus is on if not background.autobackupenabled(repo): timestamp = background.autobackupdisableduntil(repo) if timestamp is not None: ui.write( _("background backup is currently disabled until %s\n" "so your commits are not being backed up.\n" "(run 'hg cloud enable' to turn automatic backups back on)\n" ) % util.datestr(util.makedate(int(timestamp))), notice=_("note"), ) else: ui.write( _("background backup is currently disabled so your commits are not being backed up.\n" ), notice=_("note"), ) (workspacename, usernamemigration) = workspace.currentworkspacewithusernamecheck(repo) if workspacename: subscription.check(repo) backuplock.status(repo) lastsyncstate = syncstate.SyncState(repo, workspacename) if lastsyncstate.omittedheads or lastsyncstate.omittedbookmarks: hintutil.trigger("commitcloud-old-commits", repo) if usernamemigration: hintutil.trigger("commitcloud-username-migration", repo) # Don't output the summary if a backup is currently in progress. if backuplock.islocked(repo): return unbackeduprevs = repo.revs("notbackedup()") # Count the number of changesets that haven't been backed up for 10 minutes. # If there is only one, also print out its hash. backuptime = time.time() - 10 * 60 # 10 minutes ago count = 0 singleunbackeduprev = None for rev in unbackeduprevs: if repo[rev].date()[0] <= backuptime: singleunbackeduprev = rev count += 1 if count > 0: if count > 1: ui.warn(_("%d changesets are not backed up.\n") % count, notice=_("note")) else: ui.warn( _("changeset %s is not backed up.\n") % nodemod.short(repo[singleunbackeduprev].node()), notice=_("note"), ) if workspacename: ui.warn(_("(run 'hg cloud sync' to synchronize your workspace)\n")) else: ui.warn(_("(run 'hg cloud backup' to perform a backup)\n")) ui.warn( _("(if this fails, please report to %s)\n") % ccerror.getsupportcontact(ui))
def backgroundbackup(repo, command=None, dest=None): """start background backup""" ui = repo.ui if command is not None: background_cmd = command elif workspace.currentworkspace(repo): background_cmd = ["hg", "cloud", "sync"] else: background_cmd = ["hg", "cloud", "backup"] infinitepush_bgssh = ui.config("infinitepush", "bgssh") if infinitepush_bgssh: background_cmd += ["--config", "ui.ssh=%s" % infinitepush_bgssh] # developer config: infinitepushbackup.bgdebuglocks if ui.configbool("infinitepushbackup", "bgdebuglocks"): background_cmd += ["--config", "devel.debug-lockers=true"] # developer config: infinitepushbackup.bgdebug if ui.configbool("infinitepushbackup", "bgdebug", False): background_cmd.append("--debug") if dest: background_cmd += ["--dest", dest] logfile = None logdir = ui.config("infinitepushbackup", "logdir") if logdir: # make newly created files and dirs non-writable oldumask = os.umask(0o022) try: try: # the user name from the machine username = util.getuser() except Exception: username = "******" if not _checkcommonlogdir(logdir): raise WrongPermissionsException(logdir) userlogdir = os.path.join(logdir, username) util.makedirs(userlogdir) if not _checkuserlogdir(userlogdir): raise WrongPermissionsException(userlogdir) reponame = os.path.basename(repo.sharedroot) _removeoldlogfiles(userlogdir, reponame) logfile = getlogfilename(logdir, username, reponame) except (OSError, IOError) as e: ui.debug("background backup log is disabled: %s\n" % e) except WrongPermissionsException as e: ui.debug( ( "%s directory has incorrect permission, " + "background backup logging will be disabled\n" ) % e.logdir ) finally: os.umask(oldumask) if not logfile: logfile = os.devnull with open(logfile, "a") as f: timestamp = util.datestr(util.makedate(), "%Y-%m-%d %H:%M:%S %z") fullcmd = " ".join(util.shellquote(arg) for arg in background_cmd) f.write("\n%s starting: %s\n" % (timestamp, fullcmd)) Stdio = bindings.process.Stdio out = Stdio.open(logfile, append=True, create=True) bindings.process.Command.new(background_cmd[0]).args( background_cmd[1:] ).avoidinherithandles().newsession().stdin(Stdio.null()).stdout(out).stderr( out ).spawn()
def _logdate(repo, tr): revstring = " ".join(str(x) for x in util.makedate()) return writelog(repo, tr, "date.i", revstring)
def _applycloudchanges(repo, remotepath, lastsyncstate, cloudrefs, maxage, state, tr): pullcmd, pullopts = ccutil.getcommandandoptions("pull|pul") try: remotenames = extensions.find("remotenames") except KeyError: remotenames = None # Pull all the new heads and any bookmark hashes we don't have. We need to # filter cloudrefs before pull as pull doesn't check if a rev is present # locally. unfi = repo.unfiltered() newheads = [head for head in cloudrefs.heads if head not in unfi] if maxage is not None and maxage >= 0: mindate = time.time() - maxage * 86400 omittedheads = [ head for head in newheads if head in cloudrefs.headdates and cloudrefs.headdates[head] < mindate ] if omittedheads: repo.ui.status( _("omitting heads that are older than %d days:\n") % maxage) for head in omittedheads: headdatestr = util.datestr( util.makedate(cloudrefs.headdates[head])) repo.ui.status(_(" %s from %s\n") % (head[:12], headdatestr)) newheads = [head for head in newheads if head not in omittedheads] else: omittedheads = [] omittedbookmarks = [] newvisibleheads = None if visibility.tracking(repo): localheads = _getheads(repo) localheadsset = set(localheads) cloudheads = [ head for head in cloudrefs.heads if head not in omittedheads ] cloudheadsset = set(cloudheads) if localheadsset != cloudheadsset: oldvisibleheads = [ head for head in lastsyncstate.heads if head not in lastsyncstate.omittedheads ] newvisibleheads = util.removeduplicates(oldvisibleheads + cloudheads + localheads) toremove = { head for head in oldvisibleheads if head not in localheadsset or head not in cloudheadsset } newvisibleheads = [ head for head in newvisibleheads if head not in toremove ] remotebookmarknodes = [] newremotebookmarks = {} if _isremotebookmarkssyncenabled(repo.ui): newremotebookmarks = _processremotebookmarks(repo, cloudrefs.remotebookmarks, lastsyncstate) # Pull public commits, which remote bookmarks point to, if they are not # present locally. for node in newremotebookmarks.values(): if node not in unfi: remotebookmarknodes.append(node) try: snapshot = extensions.find("snapshot") except KeyError: snapshot = None addedsnapshots = [] removedsnapshots = [] newsnapshots = lastsyncstate.snapshots else: addedsnapshots = [ s for s in cloudrefs.snapshots if s not in lastsyncstate.snapshots ] removedsnapshots = [ s for s in lastsyncstate.snapshots if s not in cloudrefs.snapshots ] newsnapshots = cloudrefs.snapshots # TODO(alexeyqu): pull snapshots separately newheads += addedsnapshots backuplock.progresspulling(repo, [nodemod.bin(node) for node in newheads]) if remotebookmarknodes or newheads: # Partition the heads into groups we can pull together. headgroups = ([remotebookmarknodes] if remotebookmarknodes else []) + _partitionheads(newheads, cloudrefs.headdates) def disabled(*args, **kwargs): pass # Disable pulling of obsmarkers wrapobs = extensions.wrappedfunction(exchange, "_pullobsolete", disabled) # Disable pulling of bookmarks wrapbook = extensions.wrappedfunction(exchange, "_pullbookmarks", disabled) # Disable pulling of remote bookmarks if remotenames: wrapremotenames = extensions.wrappedfunction( remotenames, "pullremotenames", disabled) else: wrapremotenames = util.nullcontextmanager() # Disable automigration and prefetching of trees configoverride = repo.ui.configoverride( { ("pull", "automigrate"): False, ("treemanifest", "pullprefetchrevs"): "" }, "cloudsyncpull", ) prog = progress.bar(repo.ui, _("pulling from commit cloud"), total=len(headgroups)) with wrapobs, wrapbook, wrapremotenames, configoverride, prog: for index, headgroup in enumerate(headgroups): headgroupstr = " ".join([head[:12] for head in headgroup]) repo.ui.status(_("pulling %s\n") % headgroupstr) prog.value = (index, headgroupstr) pullopts["rev"] = headgroup pullcmd(repo.ui, repo, remotepath, **pullopts) repo.connectionpool.close() omittedbookmarks.extend( _mergebookmarks(repo, tr, cloudrefs.bookmarks, lastsyncstate)) if _isremotebookmarkssyncenabled(repo.ui): _updateremotebookmarks(repo, tr, newremotebookmarks) if snapshot: with repo.lock(), repo.transaction("sync-snapshots") as tr: repo.snapshotlist.update(tr, addnodes=addedsnapshots, removenodes=removedsnapshots) _mergeobsmarkers(repo, tr, cloudrefs.obsmarkers) if newvisibleheads is not None: visibility.setvisibleheads(repo, [nodemod.bin(n) for n in newvisibleheads]) # Obsmarker sharing is unreliable. Some of the commits that should now # be visible might be hidden still, and some commits that should be # hidden might still be visible. Create local obsmarkers to resolve # this. if obsolete.isenabled(repo, obsolete.createmarkersopt): unfi = repo.unfiltered() # Commits that are only visible in the cloud are commits that are # ancestors of the cloud heads but are hidden locally. cloudvisibleonly = list( unfi.set( "not public() & ::%ls & hidden()", [head for head in cloudrefs.heads if head not in omittedheads], )) # Commits that are only hidden in the cloud are commits that are # ancestors of the previous cloud heads that are not ancestors of the # current cloud heads, but have not been hidden or obsoleted locally. cloudhiddenonly = list( unfi.set( "(not public() & ::%ls) - (not public() & ::%ls) - hidden() - obsolete()", [ head for head in lastsyncstate.heads if head not in lastsyncstate.omittedheads ], [head for head in cloudrefs.heads if head not in omittedheads], )) if cloudvisibleonly or cloudhiddenonly: msg = _( "detected obsmarker inconsistency (fixing by obsoleting [%s] and reviving [%s])\n" ) % ( ", ".join( [nodemod.short(ctx.node()) for ctx in cloudhiddenonly]), ", ".join( [nodemod.short(ctx.node()) for ctx in cloudvisibleonly]), ) repo.ui.log("commitcloud_sync", msg) repo.ui.warn(msg) repo._commitcloudskippendingobsmarkers = True with repo.lock(): obsolete.createmarkers(repo, [(ctx, ()) for ctx in cloudhiddenonly]) obsolete.revive(cloudvisibleonly) repo._commitcloudskippendingobsmarkers = False # We have now synced the repo to the cloud version. Store this. logsyncop( repo, "from_cloud", cloudrefs.version, lastsyncstate.heads, cloudrefs.heads, lastsyncstate.bookmarks, cloudrefs.bookmarks, lastsyncstate.remotebookmarks, newremotebookmarks, lastsyncstate.snapshots, newsnapshots, ) lastsyncstate.update( tr, cloudrefs.version, cloudrefs.heads, cloudrefs.bookmarks, omittedheads, omittedbookmarks, maxage, newremotebookmarks, newsnapshots, ) # Also update backup state. These new heads are already backed up, # otherwise the server wouldn't have told us about them. state.update([nodemod.bin(head) for head in newheads], tr)