def applyone(self, repo, node, cl, patchfile, merge=False, log=False, filter=None): '''apply the patch in patchfile to the repository as a transplant''' (manifest, user, (time, timezone), files, message) = cl[:5] date = "%d %d" % (time, timezone) extra = {'transplant_source': node} if filter: (user, date, message) = self.filter(filter, cl, patchfile) if log: # we don't translate messages inserted into commits message += '\n(transplanted from %s)' % revlog.hex(node) self.ui.status(_('applying %s\n') % revlog.short(node)) self.ui.note('%s %s\n%s\n' % (user, date, message)) if not patchfile and not merge: raise util.Abort(_('can only omit patchfile if merging')) if patchfile: try: files = {} try: patch.patch(patchfile, self.ui, cwd=repo.root, files=files, eolmode=None) if not files: self.ui.warn( _('%s: empty changeset') % revlog.hex(node)) return None finally: files = patch.updatedir(self.ui, repo, files) except Exception, inst: if filter: os.unlink(patchfile) seriespath = os.path.join(self.path, 'series') if os.path.exists(seriespath): os.unlink(seriespath) p1 = repo.dirstate.parents()[0] p2 = node self.log(user, date, message, p1, p2, merge=merge) self.ui.write(str(inst) + '\n') raise util.Abort( _('Fix up the merge and run ' 'hg transplant --continue'))
def saveseries(self, revmap, merges): if not revmap: return if not os.path.isdir(self.path): os.mkdir(self.path) series = self.opener('series', 'w') for rev in sorted(revmap): series.write(revlog.hex(revmap[rev]) + '\n') if merges: series.write('# Merges\n') for m in merges: series.write(revlog.hex(m) + '\n') series.close()
def log(self, user, date, message, p1, p2, merge=False): """journal changelog metadata for later recover""" if not os.path.isdir(self.path): os.mkdir(self.path) fp = self.opener("journal", "w") fp.write("# User %s\n" % user) fp.write("# Date %s\n" % date) fp.write("# Node ID %s\n" % revlog.hex(p2)) fp.write("# Parent " + revlog.hex(p1) + "\n") if merge: fp.write("# Parent " + revlog.hex(p2) + "\n") fp.write(message.rstrip() + "\n") fp.close()
def saveseries(self, revmap, merges): if not revmap: return if not os.path.isdir(self.path): os.mkdir(self.path) series = self.opener("series", "w") for rev in sorted(revmap): series.write(revlog.hex(revmap[rev]) + "\n") if merges: series.write("# Merges\n") for m in merges: series.write(revlog.hex(m) + "\n") series.close()
def log(self, user, date, message, p1, p2, merge=False): '''journal changelog metadata for later recover''' if not os.path.isdir(self.path): os.mkdir(self.path) fp = self.opener('journal', 'w') fp.write('# User %s\n' % user) fp.write('# Date %s\n' % date) fp.write('# Node ID %s\n' % revlog.hex(p2)) fp.write('# Parent ' + revlog.hex(p1) + '\n') if merge: fp.write('# Parent ' + revlog.hex(p2) + '\n') fp.write(message.rstrip() + '\n') fp.close()
def add(self, orig, *args, **kwargs): origself, m, transaction, link, p1, p2, added, removed = args[:8] fastcache = fastmanifestcache.getinstance(origself.opener, self.ui) p1text = None p1hexnode = revlog.hex(p1) cacheenabled = self.ui.configbool("fastmanifest", "usecache") treeenabled = self.ui.configbool("fastmanifest", "usetree") if (cacheenabled and p1hexnode in fastcache and isinstance(m, hybridmanifest) and m._incache()): p1text = fastcache[p1hexnode].text() elif treeenabled: tree = m._treemanifest() if tree is not None: p1text = origself.revision(p1) if p1text: manifest._checkforbidden(added) # combine the changed lists into one sorted iterator work = heapq.merge([(x, False) for x in added], [(x, True) for x in removed]) # TODO: potential for optimization: avoid this silly conversion to a # python array. manifestarray = bytearray(p1text) arraytext, deltatext = m.fastdelta(manifestarray, work) cachedelta = origself.rev(p1), deltatext text = util.buffer(arraytext) node = origself.addrevision( text, transaction, link, p1, p2, cachedelta) hexnode = revlog.hex(node) # Even though we may have checked 'm._incache()' above, it may have # since disappeared, since background processes could be modifying # the cache. cachedmf = m._cachedmanifest() if cachedmf: fastcache.put_inmemory(hexnode, cachedmf) self.ui.debug("[FM] wrote manifest %s\n" % (hexnode,)) else: # If neither cache could help, fallback to the normal add node = orig(*args, **kwargs) return node
def recover(self, repo, source, opts): """commit working directory using journal metadata""" node, user, date, message, parents = self.readlog() merge = False if not user or not date or not message or not parents[0]: raise util.Abort(_("transplant log file is corrupt")) parent = parents[0] if len(parents) > 1: if opts.get("parent"): parent = source.lookup(opts["parent"]) if parent not in parents: raise util.Abort(_("%s is not a parent of %s") % (short(parent), short(node))) else: merge = True extra = {"transplant_source": node} wlock = repo.wlock() try: p1, p2 = repo.dirstate.parents() if p1 != parent: raise util.Abort(_("working dir not at transplant parent %s") % revlog.hex(parent)) if merge: repo.setparents(p1, parents[1]) n = repo.commit(message, user, date, extra=extra, editor=self.editor) if not n: raise util.Abort(_("commit failed")) if not merge: self.transplants.set(n, node) self.unlog() return n, node finally: wlock.release()
def filter(self, filter, node, changelog, patchfile): """arbitrarily rewrite changeset before applying it""" self.ui.status(_("filtering %s\n") % patchfile) user, date, msg = (changelog[1], changelog[2], changelog[4]) fd, headerfile = tempfile.mkstemp(prefix="hg-transplant-") fp = os.fdopen(fd, "w") fp.write("# HG changeset patch\n") fp.write("# User %s\n" % user) fp.write("# Date %d %d\n" % date) fp.write(msg + "\n") fp.close() try: util.system( "%s %s %s" % (filter, util.shellquote(headerfile), util.shellquote(patchfile)), environ={"HGUSER": changelog[1], "HGREVISION": revlog.hex(node)}, onerr=util.Abort, errprefix=_("filter failed"), out=self.ui.fout, ) user, date, msg = self.parselog(file(headerfile))[1:4] finally: os.unlink(headerfile) return (user, date, msg)
def filter(self, filter, node, changelog, patchfile): '''arbitrarily rewrite changeset before applying it''' self.ui.status(_('filtering %s\n') % patchfile) user, date, msg = (changelog[1], changelog[2], changelog[4]) fd, headerfile = tempfile.mkstemp(prefix='hg-transplant-') fp = os.fdopen(fd, 'w') fp.write("# HG changeset patch\n") fp.write("# User %s\n" % user) fp.write("# Date %d %d\n" % date) fp.write(msg + '\n') fp.close() try: self.ui.system('%s %s %s' % (filter, util.shellquote(headerfile), util.shellquote(patchfile)), environ={ 'HGUSER': changelog[1], 'HGREVISION': revlog.hex(node), }, onerr=util.Abort, errprefix=_('filter failed')) user, date, msg = self.parselog(file(headerfile))[1:4] finally: os.unlink(headerfile) return (user, date, msg)
def filter(self, filter, node, changelog, patchfile): '''arbitrarily rewrite changeset before applying it''' self.ui.status(_('filtering %s\n') % patchfile) user, date, msg = (changelog[1], changelog[2], changelog[4]) fd, headerfile = tempfile.mkstemp(prefix='hg-transplant-') fp = os.fdopen(fd, 'w') fp.write("# HG changeset patch\n") fp.write("# User %s\n" % user) fp.write("# Date %d %d\n" % date) fp.write(msg + '\n') fp.close() try: util.system('%s %s %s' % (filter, util.shellquote(headerfile), util.shellquote(patchfile)), environ={'HGUSER': changelog[1], 'HGREVISION': revlog.hex(node), }, onerr=util.Abort, errprefix=_('filter failed'), out=self.ui.fout) user, date, msg = self.parselog(file(headerfile))[1:4] finally: os.unlink(headerfile) return (user, date, msg)
def applyone(self, repo, node, cl, patchfile, merge=False, log=False, filter=None): '''apply the patch in patchfile to the repository as a transplant''' (manifest, user, (time, timezone), files, message) = cl[:5] date = "%d %d" % (time, timezone) extra = {'transplant_source': node} if filter: (user, date, message) = self.filter(filter, node, cl, patchfile) if log: # we don't translate messages inserted into commits message += '\n(transplanted from %s)' % revlog.hex(node) self.ui.status(_('applying %s\n') % short(node)) self.ui.note('%s %s\n%s\n' % (user, date, message)) if not patchfile and not merge: raise util.Abort(_('can only omit patchfile if merging')) if patchfile: try: files = set() patch.patch(self.ui, repo, patchfile, files=files, eolmode=None) files = list(files) except Exception, inst: seriespath = os.path.join(self.path, 'series') if os.path.exists(seriespath): os.unlink(seriespath) p1 = repo.dirstate.p1() p2 = node self.log(user, date, message, p1, p2, merge=merge) self.ui.write(str(inst) + '\n') raise TransplantError(_('fix up the merge and run ' 'hg transplant --continue'))
def recover(self, repo): '''commit working directory using journal metadata''' node, user, date, message, parents = self.readlog() merge = len(parents) == 2 if not user or not date or not message or not parents[0]: raise util.Abort(_('transplant log file is corrupt')) extra = {'transplant_source': node} wlock = repo.wlock() try: p1, p2 = repo.dirstate.parents() if p1 != parents[0]: raise util.Abort( _('working dir not at transplant parent %s') % revlog.hex(parents[0])) if merge: repo.dirstate.setparents(p1, parents[1]) n = repo.commit(None, message, user, date, extra=extra) if not n: raise util.Abort(_('commit failed')) if not merge: self.transplants.set(n, node) self.unlog() return n, node finally: del wlock
def applyone(self, repo, node, cl, patchfile, merge=False, log=False, filter=None): '''apply the patch in patchfile to the repository as a transplant''' (manifest, user, (time, timezone), files, message) = cl[:5] date = "%d %d" % (time, timezone) extra = {'transplant_source': node} if filter: (user, date, message) = self.filter(filter, cl, patchfile) if log: message += '\n(transplanted from %s)' % revlog.hex(node) self.ui.status(_('applying %s\n') % revlog.short(node)) self.ui.note('%s %s\n%s\n' % (user, date, message)) if not patchfile and not merge: raise util.Abort(_('can only omit patchfile if merging')) if patchfile: try: files = {} try: fuzz = patch.patch(patchfile, self.ui, cwd=repo.root, files=files) if not files: self.ui.warn(_('%s: empty changeset') % revlog.hex(node)) return None finally: files = patch.updatedir(self.ui, repo, files) except Exception, inst: if filter: os.unlink(patchfile) seriespath = os.path.join(self.path, 'series') if os.path.exists(seriespath): os.unlink(seriespath) p1 = repo.dirstate.parents()[0] p2 = node self.log(user, date, message, p1, p2, merge=merge) self.ui.write(str(inst) + '\n') raise util.Abort(_('Fix up the merge and run hg transplant --continue'))
def applyone(self, repo, node, cl, patchfile, merge=False, log=False, filter=None): '''apply the patch in patchfile to the repository as a transplant''' (manifest, user, (time, timezone), files, message) = cl[:5] date = "%d %d" % (time, timezone) extra = {'transplant_source': node} if filter: (user, date, message) = self.filter(filter, node, cl, patchfile) if log: # we don't translate messages inserted into commits message += '\n(transplanted from %s)' % revlog.hex(node) self.ui.status(_('applying %s\n') % short(node)) self.ui.note('%s %s\n%s\n' % (user, date, message)) if not patchfile and not merge: raise error.Abort(_('can only omit patchfile if merging')) if patchfile: try: files = set() patch.patch(self.ui, repo, patchfile, files=files, eolmode=None) files = list(files) except Exception as inst: seriespath = os.path.join(self.path, 'series') if os.path.exists(seriespath): os.unlink(seriespath) p1 = repo.dirstate.p1() p2 = node self.log(user, date, message, p1, p2, merge=merge) self.ui.write(str(inst) + '\n') raise TransplantError(_('fix up the merge and run ' 'hg transplant --continue')) else: files = None if merge: p1, p2 = repo.dirstate.parents() repo.setparents(p1, node) m = match.always(repo.root, '') else: m = match.exact(repo.root, '', files) n = repo.commit(message, user, date, extra=extra, match=m, editor=self.getcommiteditor()) if not n: self.ui.warn(_('skipping emptied changeset %s\n') % short(node)) return None if not merge: self.transplants.set(n, node) return n
def fastdelta(mf, mfgetter, base, changes): """Given a base manifest text as an array.array and a list of changes relative to that text, compute a delta that can be used by revlog. """ delta = [] dstart = None dend = None dline = [""] start = 0 # zero copy representation of base as a buffer addbuf = util.buffer(base) changes = list(changes) if len(changes) < 1000: # start with a readonly loop that finds the offset of # each line and creates the deltas for f, todelete in changes: # bs will either be the index of the item or the insert point start, end = manifest._msearch(addbuf, f, start) if not todelete: h, fl = mfgetter(f) l = "%s\0%s%s\n" % (f, revlog.hex(h), fl) else: if start == end: # item we want to delete was not found, error out raise AssertionError( (("failed to remove %s from manifest") % f)) l = "" if dstart is not None and dstart <= start and dend >= start: if dend < end: dend = end if l: dline.append(l) else: if dstart is not None: delta.append([dstart, dend, "".join(dline)]) dstart = start dend = end dline = [l] if dstart is not None: delta.append([dstart, dend, "".join(dline)]) # apply the delta to the base, and get a delta for addrevision deltatext, arraytext = manifest._addlistdelta(base, delta) else: # For large changes, it's much cheaper to just build the text and # diff it. arraytext = bytearray(mf.text()) deltatext = mdiff.textdiff(util.buffer(base), util.buffer(arraytext)) return arraytext, deltatext
def recover(self, repo, source, opts): '''commit working directory using journal metadata''' node, user, date, message, parents = self.readlog() merge = False if not user or not date or not message or not parents[0]: raise error.Abort(_('transplant log file is corrupt')) parent = parents[0] if len(parents) > 1: if opts.get('parent'): parent = source.lookup(opts['parent']) if parent not in parents: raise error.Abort( _('%s is not a parent of %s') % (short(parent), short(node))) else: merge = True extra = {'transplant_source': node} try: p1, p2 = repo.dirstate.parents() if p1 != parent: raise error.Abort( _('working directory not at transplant ' 'parent %s') % revlog.hex(parent)) if merge: repo.setparents(p1, parents[1]) modified, added, removed, deleted = repo.status()[:4] if merge or modified or added or removed or deleted: n = repo.commit(message, user, date, extra=extra, editor=self.getcommiteditor()) if not n: raise error.Abort(_('commit failed')) if not merge: self.transplants.set(n, node) else: n = None self.unlog() return n, node finally: # TODO: get rid of this meaningless try/finally enclosing. # this is kept only to reduce changes in a patch. pass
def recover(self, repo, source, opts): '''commit working directory using journal metadata''' node, user, date, message, parents = self.readlog() merge = False if not user or not date or not message or not parents[0]: raise error.Abort(_('transplant log file is corrupt')) parent = parents[0] if len(parents) > 1: if opts.get('parent'): parent = source.lookup(opts['parent']) if parent not in parents: raise error.Abort(_('%s is not a parent of %s') % (short(parent), short(node))) else: merge = True extra = {'transplant_source': node} try: p1, p2 = repo.dirstate.parents() if p1 != parent: raise error.Abort(_('working directory not at transplant ' 'parent %s') % revlog.hex(parent)) if merge: repo.setparents(p1, parents[1]) modified, added, removed, deleted = repo.status()[:4] if merge or modified or added or removed or deleted: n = repo.commit(message, user, date, extra=extra, editor=self.getcommiteditor()) if not n: raise error.Abort(_('commit failed')) if not merge: self.transplants.set(n, node) else: n = None self.unlog() return n, node finally: # TODO: get rid of this meaningless try/finally enclosing. # this is kept only to reduce changes in a patch. pass
def recover(self, repo, source, opts): '''commit working directory using journal metadata''' node, user, date, message, parents = self.readlog() merge = False if not user or not date or not message or not parents[0]: raise util.Abort(_('transplant log file is corrupt')) parent = parents[0] if len(parents) > 1: if opts.get('parent'): parent = source.lookup(opts['parent']) if parent not in parents: raise util.Abort( _('%s is not a parent of %s') % (short(parent), short(node))) else: merge = True extra = {'transplant_source': node} wlock = repo.wlock() try: p1, p2 = repo.dirstate.parents() if p1 != parent: raise util.Abort( _('working dir not at transplant parent %s') % revlog.hex(parent)) if merge: repo.setparents(p1, parents[1]) n = repo.commit(message, user, date, extra=extra, editor=self.getcommiteditor()) if not n: raise util.Abort(_('commit failed')) if not merge: self.transplants.set(n, node) self.unlog() return n, node finally: wlock.release()
def cachemanifestlist(ui, repo): cache = fastmanifestcache.getinstance(repo.store.opener, ui) total, numentries = cache.ondiskcache.totalsize(silent=False) ui.status(("cache size is: %s\n" % util.bytecount(total))) ui.status(("number of entries is: %s\n" % numentries)) if ui.debug: revs = set(repo.revs("fastmanifestcached()")) import collections revstoman = collections.defaultdict(list) for r in revs: mannode = revlog.hex(repo.changelog.changelogrevision(r).manifest) revstoman[mannode].append(str(r)) if revs: ui.status(("Most relevant cache entries appear first\n")) ui.status(("="*80)) ui.status(("\nmanifest node |revs\n")) for h in cache.ondiskcache: l = h.replace("fast","") ui.status("%s|%s\n" % (l, ",".join(revstoman.get(l,[]))))
def __init__(self, ui, opener, manifestlog, flat=None, fast=None, loadflat=None, tree=None, node=None): self.__flatmanifest = flat self.loadflat = loadflat if supportsctree and ui.configbool("fastmanifest", "usetree"): self.__treemanifest = tree else: self.__treemanifest = False if ui.configbool("fastmanifest", "usecache"): self.__cachedmanifest = fast else: self.__cachedmanifest = False assert (self.__flatmanifest is not None or self.__cachedmanifest not in (None, False) or self.__treemanifest not in (None, False) or self.loadflat is not None) self.ui = ui self.opener = opener self.manifestlog = manifestlog self.node = node self.basemanifest = None self.cachekey = revlog.hex(self.node) if self.node is not None else None self.fastcache = fastmanifestcache.getinstance(opener, self.ui) self.treecache = treemanifestcache.getinstance(opener, self.ui) self.debugfastmanifest = self.ui.configbool("fastmanifest", "debugfastmanifest", False) self.incache = (True if self.__cachedmanifest not in (None, False) else None) if self.ui.configbool("fastmanifest", "silent"): self.debug = _silent_debug else: self.debug = self.ui.debug
def applyone(self, repo, node, cl, patchfile, merge=False, log=False, filter=None): '''apply the patch in patchfile to the repository as a transplant''' (manifest, user, (time, timezone), files, message) = cl[:5] date = "%d %d" % (time, timezone) extra = {'transplant_source': node} if filter: (user, date, message) = self.filter(filter, node, cl, patchfile) if log: # we don't translate messages inserted into commits message += '\n(transplanted from %s)' % revlog.hex(node) self.ui.status(_('applying %s\n') % short(node)) self.ui.note('%s %s\n%s\n' % (user, date, message)) if not patchfile and not merge: raise error.Abort(_('can only omit patchfile if merging')) if patchfile: try: files = set() patch.patch(self.ui, repo, patchfile, files=files, eolmode=None) files = list(files) except Exception as inst: seriespath = os.path.join(self.path, 'series') if os.path.exists(seriespath): os.unlink(seriespath) p1 = repo.dirstate.p1() p2 = node self.log(user, date, message, p1, p2, merge=merge) self.ui.write(str(inst) + '\n') raise TransplantError( _('fix up the merge and run ' 'hg transplant --continue')) else: files = None if merge: p1, p2 = repo.dirstate.parents() repo.setparents(p1, node) m = match.always(repo.root, '') else: m = match.exact(repo.root, '', files) n = repo.commit(message, user, date, extra=extra, match=m, editor=self.getcommiteditor()) if not n: self.ui.warn(_('skipping emptied changeset %s\n') % short(node)) return None if not merge: self.transplants.set(n, node) return n
def revstr(rev): if rev == 'HEAD': rev = 'tip' return revlog.hex(repo.lookup(rev))
def kwtransplanted(repo, ctx, **args): """:transplanted: String. The node identifier of the transplanted changeset if any.""" n = ctx.extra().get('transplant_source') return n and revlog.hex(n) or ''
def cachemanifestfillandtrim(ui, repo, revset): """Cache the manifests described by `revset`. This priming is subject to limits imposed by the cache, and thus not all the entries may be written. """ try: with concurrency.looselock(repo.vfs, "fastmanifest", constants.WORKER_SPAWN_LOCK_STEAL_TIMEOUT): cache = fastmanifestcache.getinstance(repo.store.opener, ui) computedrevs = scmutil.revrange(repo, revset) sortedrevs = sorted(computedrevs, key=lambda x:-x) repo.ui.log("fastmanifest", "FM: trying to cache %s\n" % str(sortedrevs)) if len(sortedrevs) == 0: # normally, we prune as we make space for new revisions to add # to the cache. however, if we're not adding any new elements, # we'll never check the disk cache size. this is an explicit # check for that particular scenario. before = cache.ondiskcache.items() cache.prune() after = cache.ondiskcache.items() diff = set(after) - set(before) if diff: ui.log("fastmanifest", "FM: removed entries %s\n" % str(diff)) else: ui.log("fastmanifest", "FM: no entries removed\n") else: revstomannodes = {} mannodesprocessed = set() for rev in sortedrevs: mannode = revlog.hex( repo.changelog.changelogrevision(rev).manifest) revstomannodes[rev] = mannode mannodesprocessed.add(mannode) if mannode in cache.ondiskcache: ui.debug("[FM] skipped %s, already cached " "(fast path)\n" % (mannode,)) repo.ui.log("fastmanifest", "FM: skip(rev, man) %s->%s\n" % (rev, mannode)) # Account for the fact that we access this manifest cache.ondiskcache.touch(mannode) continue manifest = repo[rev].manifest() fastmanifest = cfastmanifest.fastmanifest(manifest.text()) cache.makeroomfor(fastmanifest.bytes(), mannodesprocessed) try: cache[mannode] = fastmanifest repo.ui.log("fastmanifest", "FM: cached(rev,man) " "%s->%s\n" % (rev, mannode)) except CacheFullException: repo.ui.log("fastmanifest", "FM: overflow\n") break # Make the least relevant entries have an artificially older # mtime than the more relevant ones. We use a resolution of 2 # for time to work accross all platforms and ensure that the # order is marked. # # Note that we use sortedrevs and not revs because here we # don't care about the shuffling, we just want the most relevant # revisions to have more recent mtime. mtimemultiplier = 2 for offset, rev in enumerate(sortedrevs): if rev in revstomannodes: hexnode = revstomannodes[rev] cache.ondiskcache.touch(hexnode, delay=offset * mtimemultiplier) else: metricscollector.get().recordsample("cacheoverflow", hit=True) # We didn't have enough space for that rev except error.LockHeld: return except (OSError, IOError) as ex: if ex.errno == errno.EACCES: # permission issue ui.warn(("warning: not using fastmanifest\n")) ui.warn(("(make sure that .hg/store is writeable)\n")) return raise total, numentries = cache.ondiskcache.totalsize() if isinstance(cache.limit, _systemawarecachelimit): free = cache.limit.free / 1024**2 else: free = -1 metricscollector.get().recordsample("ondiskcachestats", bytes=total, numentries=numentries, limit=(cache.limit.bytes() / 1024**2), freespace=free)