def create_bundle(store, commits, bundle2caps={}):
version = '01'
chunk_type = RawRevChunk01
if bundle2caps:
versions = bundle2caps.get('changegroup')
if versions:
if '02' in versions:
chunk_type = RawRevChunk02
version = '02'
cg = create_changegroup(store, bundle_data(store, commits), chunk_type)
if bundle2caps:
from mercurial.util import chunkbuffer
yield 'HG20'
yield '\0' * 4 # bundle parameters length: no params
replycaps = bundle2caps.get('replycaps')
if replycaps:
for chunk in bundlepart('REPLYCAPS', data=chunkbuffer(replycaps)):
yield chunk
for chunk in bundlepart('CHANGEGROUP',
advisoryparams=(('version', version),),
data=chunkbuffer(cg)):
yield chunk
yield '\0' * 4 # End of bundle
else:
for chunk in cg:
yield chunk
def bundle2scratchbranch(op, part):
'''unbundle a bundle2 part containing a changegroup to store'''
bundler = bundle2.bundle20(op.repo.ui)
cgversion = part.params.get('cgversion', '01')
cgpart = bundle2.bundlepart('changegroup', data=part.read())
cgpart.addparam('version', cgversion)
bundler.addpart(cgpart)
buf = util.chunkbuffer(bundler.getchunks())
fd, bundlefile = tempfile.mkstemp()
try:
try:
fp = os.fdopen(fd, r'wb')
fp.write(buf.read())
finally:
fp.close()
storebundle(op, part.params, bundlefile)
finally:
try:
os.unlink(bundlefile)
except OSError as e:
if e.errno != errno.ENOENT:
raise
return 1
def create_bundle(store, commits, bundle2caps={}):
version = '01'
chunk_type = RawRevChunk01
if bundle2caps:
versions = bundle2caps.get('changegroup')
if versions:
if '02' in versions:
chunk_type = RawRevChunk02
version = '02'
cg = create_changegroup(store, bundle_data(store, commits), chunk_type)
if bundle2caps:
from mercurial.util import chunkbuffer
yield 'HG20'
yield '\0' * 4 # bundle parameters length: no params
if bundle2caps.get('replycaps'):
for chunk in bundlepart('REPLYCAPS'):
yield chunk
for chunk in bundlepart('CHANGEGROUP',
advisoryparams=(('version', version),),
data=chunkbuffer(cg)):
yield chunk
yield '\0' * 4 # End of bundle
else:
for chunk in cg:
yield chunk
def push(repo, store, what, repo_heads, repo_branches):
fast_import = FastImport()
store.init_fast_import(fast_import)
def heads():
for sha1 in store.heads(repo_branches):
yield '^%s' % store.changeset_ref(sha1)
def local_bases():
for c in Git.iter('rev-list',
'--stdin',
'--topo-order',
'--full-history',
'--boundary',
*(w for w in what if w),
stdin=heads()):
if c[0] != '-':
continue
yield store.hg_changeset(c[1:])
for w in what:
rev = store.hg_changeset(w)
if rev:
yield rev
common = findcommon(repo, store, set(local_bases()))
logging.info('common: %s' % common)
def revs():
for sha1 in common:
yield '^%s' % store.changeset_ref(sha1)
push_commits = list(
Git.iter('rev-list',
'--stdin',
'--topo-order',
'--full-history',
'--parents',
'--reverse',
*(w for w in what if w),
stdin=revs()))
pushed = False
if push_commits:
chunks = util.chunkbuffer(create_bundle(store, push_commits))
cg = cg1unpacker(chunks, 'UN')
if all(v[1] for v in what.values()):
repo_heads = ['force']
else:
repo_heads = [unhexlify(h) for h in repo_heads]
if repo.local():
repo.local().ui.setconfig('server', 'validate', True)
pushed = repo.unbundle(cg, repo_heads, '') != 0
return gitdag(push_commits) if pushed else ()
def storetobundlestore(orig, repo, op, unbundler):
"""stores the incoming bundle coming from push command to the bundlestore
instead of applying on the revlogs"""
repo.ui.status(_(b"storing changesets on the bundlestore\n"))
bundler = bundle2.bundle20(repo.ui)
# processing each part and storing it in bundler
with bundle2.partiterator(repo, op, unbundler) as parts:
for part in parts:
bundlepart = None
if part.type == b'replycaps':
# This configures the current operation to allow reply parts.
bundle2._processpart(op, part)
else:
bundlepart = bundle2.bundlepart(part.type, data=part.read())
for key, value in pycompat.iteritems(part.params):
bundlepart.addparam(key, value)
# Certain parts require a response
if part.type in (b'pushkey', b'changegroup'):
if op.reply is not None:
rpart = op.reply.newpart(b'reply:%s' % part.type)
rpart.addparam(
b'in-reply-to', b'%d' % part.id, mandatory=False
)
rpart.addparam(b'return', b'1', mandatory=False)
op.records.add(
part.type,
{
b'return': 1,
},
)
if bundlepart:
bundler.addpart(bundlepart)
# storing the bundle in the bundlestore
buf = util.chunkbuffer(bundler.getchunks())
fd, bundlefile = pycompat.mkstemp()
try:
try:
fp = os.fdopen(fd, 'wb')
fp.write(buf.read())
finally:
fp.close()
storebundle(op, {}, bundlefile)
finally:
try:
os.unlink(bundlefile)
except Exception:
# we would rather see the original exception
pass
def push(repo, store, what, repo_heads, repo_branches):
store.init_fast_import()
def heads():
for sha1 in store.heads(repo_branches):
yield '^%s' % store.changeset_ref(sha1)
def local_bases():
for c in Git.iter('rev-list', '--stdin', '--topo-order',
'--full-history', '--boundary',
*(w for w in what if w), stdin=heads()):
if c[0] != '-':
continue
yield store.hg_changeset(c[1:])
for w in what:
rev = store.hg_changeset(w)
if rev:
yield rev
common = findcommon(repo, store, set(local_bases()))
logging.info('common: %s' % common)
def revs():
for sha1 in common:
yield '^%s' % store.changeset_ref(sha1)
push_commits = list(Git.iter('rev-list', '--stdin', '--topo-order',
'--full-history', '--parents', '--reverse',
*(w for w in what if w), stdin=revs()))
pushed = False
if push_commits:
has_root = any(len(p) == 40 for p in push_commits)
force = all(v[1] for v in what.values())
if has_root and repo_heads:
if not force:
raise Exception('Cannot push a new root')
else:
logging.warn('Pushing a new root')
chunks = util.chunkbuffer(create_bundle(store, push_commits))
cg = cg1unpacker(chunks, 'UN')
if force:
repo_heads = ['force']
else:
if not repo_heads:
repo_heads = [NULL_NODE_ID]
repo_heads = [unhexlify(h) for h in repo_heads]
if repo.local():
repo.local().ui.setconfig('server', 'validate', True)
pushed = repo.unbundle(cg, repo_heads, '') != 0
return gitdag(push_commits) if pushed else ()
def _rebundle(bundlerepo, bundleroots, unknownhead):
'''
Bundle may include more revision then user requested. For example,
if user asks for revision but bundle also consists its descendants.
This function will filter out all revision that user is not requested.
'''
parts = []
version = '02'
outgoing = discovery.outgoing(bundlerepo, commonheads=bundleroots,
missingheads=[unknownhead])
cgstream = changegroup.makestream(bundlerepo, outgoing, version, 'pull')
cgstream = util.chunkbuffer(cgstream).read()
cgpart = bundle2.bundlepart('changegroup', data=cgstream)
cgpart.addparam('version', version)
parts.append(cgpart)
return parts
def bundle2phaseheads(orig, op, part):
# Merges many dicts into one. First it converts them to list of pairs,
# then concatenates them (using sum), and then creates a diff out of them.
replacements = dict(sum([record.items()
for record
in op.records[rebaseparttype]],
[]))
decodedphases = phasesmod.binarydecode(part)
replacedphases = []
for phasetype in decodedphases:
replacedphases.append(
[replacements.get(node, node) for node in phasetype])
# Since we've just read the bundle part, then `orig()` won't be able to
# read it again. Let's replace payload stream with new stream of replaced
# nodes.
part._payloadstream = util.chunkbuffer(
[phasesmod.binaryencode(replacedphases)])
return orig(op, part)
def writerevs(ui, r1, r2, order, tr):
ui.status(_('writing revs\n'))
count = [0]
def progress(*args):
ui.progress(_('writing'), count[0], total=len(order))
count[0] += 1
order = [r1.node(r) for r in order]
# this is a bit ugly, but it works
lookup = lambda x: "%020d" % r1.linkrev(r1.rev(x))
unlookup = lambda x: int(x, 10)
try:
group = util.chunkbuffer(r1.group(order, lookup, progress))
group = changegroup.unbundle10(group, "UN")
r2.addgroup(group, unlookup, tr)
finally:
ui.progress(_('writing'), None)
def writerevs(ui, r1, r2, order, tr):
ui.status(_('writing revs\n'))
count = [0]
def progress(*args):
ui.progress(_('writing'), count[0], total=len(order))
count[0] += 1
order = [r1.node(r) for r in order]
# this is a bit ugly, but it works
lookup = lambda x: "%020d" % r1.linkrev(r1.rev(x))
unlookup = lambda x: int(x, 10)
try:
group = util.chunkbuffer(r1.group(order, lookup, progress))
group = changegroup.unbundle10(group, "UN")
r2.addgroup(group, unlookup, tr)
finally:
ui.progress(_('writing'), None)
def push(repo, store, what, repo_heads, repo_branches, dry_run=False):
def heads():
for sha1 in store.heads(repo_branches):
yield '^%s' % store.changeset_ref(sha1)
def local_bases():
h = chain(heads(), (w for w in what if w))
for c, t, p in GitHgHelper.rev_list('--topo-order', '--full-history',
'--boundary', *h):
if c[0] != '-':
continue
yield store.hg_changeset(c[1:])
for w in what:
rev = store.hg_changeset(w)
if rev:
yield rev
common = findcommon(repo, store, set(local_bases()))
logging.info('common: %s', common)
def revs():
for sha1 in common:
yield '^%s' % store.changeset_ref(sha1)
revs = chain(revs(), (w for w in what if w))
push_commits = list((c, p) for c, t, p in GitHgHelper.rev_list(
'--topo-order', '--full-history', '--parents', '--reverse', *revs))
pushed = False
if push_commits:
has_root = any(len(p) == 40 for p in push_commits)
force = all(v[1] for v in what.values())
if has_root and repo_heads:
if not force:
raise Exception('Cannot push a new root')
else:
logging.warn('Pushing a new root')
if force:
repo_heads = ['force']
else:
if not repo_heads:
repo_heads = [NULL_NODE_ID]
repo_heads = [unhexlify(h) for h in repo_heads]
if push_commits and not dry_run:
if repo.local():
repo.local().ui.setconfig('server', 'validate', True)
b2caps = bundle2caps(repo) if unbundle20 else {}
logging.getLogger('bundle2').debug('%r', b2caps)
if b2caps:
b2caps['replycaps'] = encodecaps({'error': ['abort']})
cg = create_bundle(store, push_commits, b2caps)
if not isinstance(repo, HelperRepo):
cg = util.chunkbuffer(cg)
if not b2caps:
cg = cg1unpacker(cg, 'UN')
reply = repo.unbundle(cg, repo_heads, '')
if unbundle20 and isinstance(reply, unbundle20):
parts = iter(reply.iterparts())
for part in parts:
logging.getLogger('bundle2').debug('part: %s', part.type)
logging.getLogger('bundle2').debug('params: %r', part.params)
if part.type == 'output':
sys.stderr.write(part.read())
elif part.type == 'reply:changegroup':
# TODO: should check params['in-reply-to']
reply = int(part.params['return'])
elif part.type == 'error:abort':
raise error.Abort(part.params['message'],
hint=part.params.get('hint'))
else:
logging.getLogger('bundle2').warning(
'ignoring bundle2 part: %s', part.type)
pushed = reply != 0
return gitdag(push_commits) if pushed or dry_run else ()
def processparts(orig, repo, op, unbundler):
# make sure we don't wrap processparts in case of `hg unbundle`
if op.source == 'unbundle':
return orig(repo, op, unbundler)
# this server routes each push to bundle store
if repo.ui.configbool('infinitepush', 'pushtobundlestore'):
return storetobundlestore(orig, repo, op, unbundler)
if unbundler.params.get('infinitepush') != 'True':
return orig(repo, op, unbundler)
handleallparts = repo.ui.configbool('infinitepush', 'storeallparts')
bundler = bundle2.bundle20(repo.ui)
cgparams = None
with bundle2.partiterator(repo, op, unbundler) as parts:
for part in parts:
bundlepart = None
if part.type == 'replycaps':
# This configures the current operation to allow reply parts.
bundle2._processpart(op, part)
elif part.type == bundleparts.scratchbranchparttype:
# Scratch branch parts need to be converted to normal
# changegroup parts, and the extra parameters stored for later
# when we upload to the store. Eventually those parameters will
# be put on the actual bundle instead of this part, then we can
# send a vanilla changegroup instead of the scratchbranch part.
cgversion = part.params.get('cgversion', '01')
bundlepart = bundle2.bundlepart('changegroup',
data=part.read())
bundlepart.addparam('version', cgversion)
cgparams = part.params
# If we're not dumping all parts into the new bundle, we need to
# alert the future pushkey and phase-heads handler to skip
# the part.
if not handleallparts:
op.records.add(scratchbranchparttype + '_skippushkey',
True)
op.records.add(scratchbranchparttype + '_skipphaseheads',
True)
else:
if handleallparts:
# Ideally we would not process any parts, and instead just
# forward them to the bundle for storage, but since this
# differs from previous behavior, we need to put it behind a
# config flag for incremental rollout.
bundlepart = bundle2.bundlepart(part.type,
data=part.read())
for key, value in part.params.iteritems():
bundlepart.addparam(key, value)
# Certain parts require a response
if part.type == 'pushkey':
if op.reply is not None:
rpart = op.reply.newpart('reply:pushkey')
rpart.addparam('in-reply-to',
str(part.id),
mandatory=False)
rpart.addparam('return', '1', mandatory=False)
else:
bundle2._processpart(op, part)
if handleallparts:
op.records.add(part.type, {
'return': 1,
})
if bundlepart:
bundler.addpart(bundlepart)
# If commits were sent, store them
if cgparams:
buf = util.chunkbuffer(bundler.getchunks())
fd, bundlefile = tempfile.mkstemp()
try:
try:
fp = os.fdopen(fd, r'wb')
fp.write(buf.read())
finally:
fp.close()
storebundle(op, cgparams, bundlefile)
finally:
try:
os.unlink(bundlefile)
except Exception:
# we would rather see the original exception
pass
def getlfile(ui, hash):
return util.chunkbuffer(openstore(ui=ui)._get(hash))
def push(repo, store, what, repo_heads, repo_branches):
store.init_fast_import()
def heads():
for sha1 in store.heads(repo_branches):
yield '^%s' % store.changeset_ref(sha1)
def local_bases():
for c in Git.iter('rev-list', '--stdin', '--topo-order',
'--full-history', '--boundary',
*(w for w in what if w), stdin=heads()):
if c[0] != '-':
continue
yield store.hg_changeset(c[1:])
for w in what:
rev = store.hg_changeset(w)
if rev:
yield rev
common = findcommon(repo, store, set(local_bases()))
logging.info('common: %s' % common)
def revs():
for sha1 in common:
yield '^%s' % store.changeset_ref(sha1)
push_commits = list(Git.iter('rev-list', '--stdin', '--topo-order',
'--full-history', '--parents', '--reverse',
*(w for w in what if w), stdin=revs()))
pushed = False
if push_commits:
has_root = any(len(p) == 40 for p in push_commits)
force = all(v[1] for v in what.values())
if has_root and repo_heads:
if not force:
raise Exception('Cannot push a new root')
else:
logging.warn('Pushing a new root')
if force:
repo_heads = ['force']
else:
if not repo_heads:
repo_heads = [NULL_NODE_ID]
repo_heads = [unhexlify(h) for h in repo_heads]
if repo.local():
repo.local().ui.setconfig('server', 'validate', True)
b2caps = bundle2caps(repo) if unbundle20 else {}
if b2caps and (repo.url().startswith(('http://', 'https://')) or
not isinstance(repo, HelperRepo)):
b2caps['replycaps'] = True
cg = create_bundle(store, push_commits, b2caps)
if not isinstance(repo, HelperRepo):
cg = util.chunkbuffer(cg)
if not b2caps:
cg = cg1unpacker(cg, 'UN')
reply = repo.unbundle(cg, repo_heads, '')
if unbundle20 and isinstance(reply, unbundle20):
parts = iter(reply.iterparts())
for part in parts:
if part.type == 'output':
sys.stderr.write(part.read())
elif part.type == 'reply:changegroup':
# TODO: should check params['in-reply-to']
reply = int(part.params['return'])
else:
logging.getLogger('bundle2').warning(
'ignoring bundle2 part: %s', part.type)
pushed = reply != 0
return gitdag(push_commits) if pushed else ()
