def perfchangegroupchangelog(ui, repo, version='02', rev=None, **opts):
"""Benchmark producing a changelog group for a changegroup.
This measures the time spent processing the changelog during a
bundle operation. This occurs during `hg bundle` and on a server
processing a `getbundle` wire protocol request (handles clones
and pull requests).
By default, all revisions are added to the changegroup.
"""
cl = repo.changelog
revs = [cl.lookup(r) for r in repo.revs(rev or 'all()')]
bundler = changegroup.getbundler(version, repo)
def lookup(node):
# The real bundler reads the revision in order to access the
# manifest node and files list. Do that here.
cl.read(node)
return node
def d():
for chunk in bundler.group(revs, cl, lookup):
pass
timer, fm = gettimer(ui, opts)
timer(d)
fm.end()
def generate_ellipses_bundle2_for_widening(
bundler,
repo,
oldmatch,
newmatch,
version,
common,
known,
):
common = set(common or [nullid])
# Steps:
# 1. Send kill for "$known & ::common"
#
# 2. Send changegroup for ::common
#
# 3. Proceed.
#
# In the future, we can send kills for only the specific
# nodes we know should go away or change shape, and then
# send a data stream that tells the client something like this:
#
# a) apply this changegroup
# b) apply nodes XXX, YYY, ZZZ that you already have
# c) goto a
#
# until they've built up the full new state.
knownrevs = {repo.changelog.rev(n) for n in known}
# TODO: we could send only roots() of this set, and the
# list of nodes in common, and the client could work out
# what to strip, instead of us explicitly sending every
# single node.
deadrevs = knownrevs
def genkills():
for r in deadrevs:
yield _KILLNODESIGNAL
yield repo.changelog.node(r)
yield _DONESIGNAL
bundler.newpart(_CHANGESPECPART, data=genkills())
newvisit, newfull, newellipsis = exchange._computeellipsis(
repo, set(), common, knownrevs, newmatch
)
if newvisit:
packer = changegroup.getbundler(
version,
repo,
matcher=newmatch,
ellipses=True,
shallow=False,
ellipsisroots=newellipsis,
fullnodes=newfull,
)
cgdata = packer.generate(common, newvisit, False, b'narrow_widen')
part = bundler.newpart(b'changegroup', data=cgdata)
part.addparam(b'version', version)
if scmutil.istreemanifest(repo):
part.addparam(b'treemanifest', b'1')
def generateellipsesbundle2(
bundler,
repo,
include,
exclude,
version,
common,
heads,
depth,
):
match = narrowspec.match(repo.root, include=include, exclude=exclude)
if depth is not None:
depth = int(depth)
if depth < 1:
raise error.Abort(_(b'depth must be positive, got %d') % depth)
heads = set(heads or repo.heads())
common = set(common or [nullid])
visitnodes, relevant_nodes, ellipsisroots = exchange._computeellipsis(
repo, common, heads, set(), match, depth=depth
)
repo.ui.debug(b'Found %d relevant revs\n' % len(relevant_nodes))
if visitnodes:
packer = changegroup.getbundler(
version,
repo,
matcher=match,
ellipses=True,
shallow=depth is not None,
ellipsisroots=ellipsisroots,
fullnodes=relevant_nodes,
)
cgdata = packer.generate(common, visitnodes, False, b'narrow_widen')
part = bundler.newpart(b'changegroup', data=cgdata)
part.addparam(b'version', version)
if scmutil.istreemanifest(repo):
part.addparam(b'treemanifest', b'1')
def _packellipsischangegroup(repo, common, match, relevant_nodes,
ellipsisroots, visitnodes, depth, source,
version):
if version in ('01', '02'):
raise error.Abort(
'ellipsis nodes require at least cg3 on client and server, '
'but negotiated version %s' % version)
# We wrap cg1packer.revchunk, using a side channel to pass
# relevant_nodes into that area. Then if linknode isn't in the
# set, we know we have an ellipsis node and we should defer
# sending that node's data. We override close() to detect
# pending ellipsis nodes and flush them.
packer = changegroup.getbundler(version, repo)
# Let the packer have access to the narrow matcher so it can
# omit filelogs and dirlogs as needed
packer._narrow_matcher = lambda: match
# Give the packer the list of nodes which should not be
# ellipsis nodes. We store this rather than the set of nodes
# that should be an ellipsis because for very large histories
# we expect this to be significantly smaller.
packer.full_nodes = relevant_nodes
# Maps ellipsis revs to their roots at the changelog level.
packer.precomputed_ellipsis = ellipsisroots
# Maps CL revs to per-revlog revisions. Cleared in close() at
# the end of each group.
packer.clrev_to_localrev = {}
packer.next_clrev_to_localrev = {}
# Maps changelog nodes to changelog revs. Filled in once
# during changelog stage and then left unmodified.
packer.clnode_to_rev = {}
packer.changelog_done = False
# If true, informs the packer that it is serving shallow content and might
# need to pack file contents not introduced by the changes being packed.
packer.is_shallow = depth is not None
return packer.generate(common, visitnodes, False, source)
def getbundlechangegrouppart_narrow(bundler,
repo,
source,
bundlecaps=None,
b2caps=None,
heads=None,
common=None,
**kwargs):
assert repo.ui.configbool('experimental', 'narrowservebrokenellipses')
cgversions = b2caps.get('changegroup')
if cgversions: # 3.1 and 3.2 ship with an empty value
cgversions = [
v for v in cgversions
if v in changegroup.supportedoutgoingversions(repo)
]
if not cgversions:
raise ValueError(_('no common changegroup version'))
version = max(cgversions)
else:
raise ValueError(
_("server does not advertise changegroup version,"
" can't negotiate support for ellipsis nodes"))
include = sorted(filter(bool, kwargs.get(r'includepats', [])))
exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
newmatch = narrowspec.match(repo.root, include=include, exclude=exclude)
depth = kwargs.get(r'depth', None)
if depth is not None:
depth = int(depth)
if depth < 1:
raise error.Abort(_('depth must be positive, got %d') % depth)
heads = set(heads or repo.heads())
common = set(common or [nullid])
oldinclude = sorted(filter(bool, kwargs.get(r'oldincludepats', [])))
oldexclude = sorted(filter(bool, kwargs.get(r'oldexcludepats', [])))
known = {bin(n) for n in kwargs.get(r'known', [])}
if known and (oldinclude != include or oldexclude != exclude):
# Steps:
# 1. Send kill for "$known & ::common"
#
# 2. Send changegroup for ::common
#
# 3. Proceed.
#
# In the future, we can send kills for only the specific
# nodes we know should go away or change shape, and then
# send a data stream that tells the client something like this:
#
# a) apply this changegroup
# b) apply nodes XXX, YYY, ZZZ that you already have
# c) goto a
#
# until they've built up the full new state.
# Convert to revnums and intersect with "common". The client should
# have made it a subset of "common" already, but let's be safe.
known = set(repo.revs("%ln & ::%ln", known, common))
# TODO: we could send only roots() of this set, and the
# list of nodes in common, and the client could work out
# what to strip, instead of us explicitly sending every
# single node.
deadrevs = known
def genkills():
for r in deadrevs:
yield _KILLNODESIGNAL
yield repo.changelog.node(r)
yield _DONESIGNAL
bundler.newpart(_CHANGESPECPART, data=genkills())
newvisit, newfull, newellipsis = exchange._computeellipsis(
repo, set(), common, known, newmatch)
if newvisit:
packer = changegroup.getbundler(version,
repo,
matcher=newmatch,
ellipses=True,
shallow=depth is not None,
ellipsisroots=newellipsis,
fullnodes=newfull)
cgdata = packer.generate(common, newvisit, False, 'narrow_widen')
part = bundler.newpart('changegroup', data=cgdata)
part.addparam('version', version)
if 'treemanifest' in repo.requirements:
part.addparam('treemanifest', '1')
visitnodes, relevant_nodes, ellipsisroots = exchange._computeellipsis(
repo, common, heads, set(), newmatch, depth=depth)
repo.ui.debug('Found %d relevant revs\n' % len(relevant_nodes))
if visitnodes:
packer = changegroup.getbundler(version,
repo,
matcher=newmatch,
ellipses=True,
shallow=depth is not None,
ellipsisroots=ellipsisroots,
fullnodes=relevant_nodes)
cgdata = packer.generate(common, visitnodes, False, 'narrow_widen')
part = bundler.newpart('changegroup', data=cgdata)
part.addparam('version', version)
if 'treemanifest' in repo.requirements:
part.addparam('treemanifest', '1')
def generateellipsesbundle2(bundler, repo, oldinclude, oldexclude, newinclude,
newexclude, version, common, heads, known, depth):
newmatch = narrowspec.match(repo.root,
include=newinclude,
exclude=newexclude)
if depth is not None:
depth = int(depth)
if depth < 1:
raise error.Abort(_('depth must be positive, got %d') % depth)
heads = set(heads or repo.heads())
common = set(common or [nullid])
if known and (oldinclude != newinclude or oldexclude != newexclude):
# Steps:
# 1. Send kill for "$known & ::common"
#
# 2. Send changegroup for ::common
#
# 3. Proceed.
#
# In the future, we can send kills for only the specific
# nodes we know should go away or change shape, and then
# send a data stream that tells the client something like this:
#
# a) apply this changegroup
# b) apply nodes XXX, YYY, ZZZ that you already have
# c) goto a
#
# until they've built up the full new state.
# Convert to revnums and intersect with "common". The client should
# have made it a subset of "common" already, but let's be safe.
known = set(repo.revs("%ln & ::%ln", known, common))
# TODO: we could send only roots() of this set, and the
# list of nodes in common, and the client could work out
# what to strip, instead of us explicitly sending every
# single node.
deadrevs = known
def genkills():
for r in deadrevs:
yield _KILLNODESIGNAL
yield repo.changelog.node(r)
yield _DONESIGNAL
bundler.newpart(_CHANGESPECPART, data=genkills())
newvisit, newfull, newellipsis = exchange._computeellipsis(
repo, set(), common, known, newmatch)
if newvisit:
packer = changegroup.getbundler(version,
repo,
matcher=newmatch,
ellipses=True,
shallow=depth is not None,
ellipsisroots=newellipsis,
fullnodes=newfull)
cgdata = packer.generate(common, newvisit, False, 'narrow_widen')
part = bundler.newpart('changegroup', data=cgdata)
part.addparam('version', version)
if 'treemanifest' in repo.requirements:
part.addparam('treemanifest', '1')
visitnodes, relevant_nodes, ellipsisroots = exchange._computeellipsis(
repo, common, heads, set(), newmatch, depth=depth)
repo.ui.debug('Found %d relevant revs\n' % len(relevant_nodes))
if visitnodes:
packer = changegroup.getbundler(version,
repo,
matcher=newmatch,
ellipses=True,
shallow=depth is not None,
ellipsisroots=ellipsisroots,
fullnodes=relevant_nodes)
cgdata = packer.generate(common, visitnodes, False, 'narrow_widen')
part = bundler.newpart('changegroup', data=cgdata)
part.addparam('version', version)
if 'treemanifest' in repo.requirements:
part.addparam('treemanifest', '1')