def getbundle(repo, source, heads=None, common=None, bundlecaps=None, **kwargs): """return a full bundle (with potentially multiple kind of parts) Could be a bundle HG10 or a bundle HG2X depending on bundlecaps passed. For now, the bundle can contain only changegroup, but this will changes when more part type will be available for bundle2. This is different from changegroup.getbundle that only returns an HG10 changegroup bundle. They may eventually get reunited in the future when we have a clearer idea of the API we what to query different data. The implementation is at a very early stage and will get massive rework when the API of bundle is refined. """ # build changegroup bundle here. cg = changegroup.getbundle(repo, source, heads=heads, common=common, bundlecaps=bundlecaps) if bundlecaps is None or 'HG2X' not in bundlecaps: return cg # very crude first implementation, # the bundle API will change and the generation will be done lazily. b2caps = {} for bcaps in bundlecaps: if bcaps.startswith('bundle2='): blob = urllib.unquote(bcaps[len('bundle2='):]) b2caps.update(bundle2.decodecaps(blob)) bundler = bundle2.bundle20(repo.ui, b2caps) if cg: part = bundle2.bundlepart('b2x:changegroup', data=cg.getchunks()) bundler.addpart(part) _getbundleextrapart(bundler, repo, source, heads=heads, common=common, bundlecaps=bundlecaps, **kwargs) return util.chunkbuffer(bundler.getchunks())
def unbundle(repo, proto, heads): their_heads = decodelist(heads) try: proto.redirect() exchange.check_heads(repo, their_heads, 'preparing changes') # write bundle data to temporary file because it can be big fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-') fp = os.fdopen(fd, 'wb+') r = 0 try: proto.getfile(fp) fp.seek(0) gen = exchange.readbundle(repo.ui, fp, None) r = exchange.unbundle(repo, gen, their_heads, 'serve', proto._client()) if util.safehasattr(r, 'addpart'): # The return looks streameable, we are in the bundle2 case and # should return a stream. return streamres(r.getchunks()) return pushres(r) finally: fp.close() os.unlink(tempname) except bundle2.UnknownPartError, exc: bundler = bundle2.bundle20(repo.ui) part = bundle2.bundlepart('B2X:ERROR:UNKNOWNPART', [('parttype', str(exc))]) bundler.addpart(part) return streamres(bundler.getchunks())
def unbundle(repo, proto, heads): their_heads = decodelist(heads) try: proto.redirect() exchange.check_heads(repo, their_heads, "preparing changes") # write bundle data to temporary file because it can be big fd, tempname = tempfile.mkstemp(prefix="hg-unbundle-") fp = os.fdopen(fd, "wb+") r = 0 try: proto.getfile(fp) fp.seek(0) gen = exchange.readbundle(repo.ui, fp, None) r = exchange.unbundle(repo, gen, their_heads, "serve", proto._client()) if util.safehasattr(r, "addpart"): # The return looks streameable, we are in the bundle2 case and # should return a stream. return streamres(r.getchunks()) return pushres(r) finally: fp.close() os.unlink(tempname) except error.BundleValueError, exc: bundler = bundle2.bundle20(repo.ui) errpart = bundler.newpart("B2X:ERROR:UNSUPPORTEDCONTENT") if exc.parttype is not None: errpart.addparam("parttype", exc.parttype) if exc.params: errpart.addparam("params", "\0".join(exc.params)) return streamres(bundler.getchunks())
def _pushbundle2(pushop): """push data to the remote using bundle2 The only currently supported type of data is changegroup but this will evolve in the future.""" bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote)) pushback = (pushop.trmanager and pushop.ui.configbool('experimental', 'bundle2.pushback')) # create reply capability capsblob = bundle2.encodecaps(bundle2.getrepocaps(pushop.repo, allowpushback=pushback)) bundler.newpart('b2x:replycaps', data=capsblob) replyhandlers = [] for partgenname in b2partsgenorder: partgen = b2partsgenmapping[partgenname] ret = partgen(pushop, bundler) if callable(ret): replyhandlers.append(ret) # do not push if nothing to push if bundler.nbparts <= 1: return stream = util.chunkbuffer(bundler.getchunks()) try: reply = pushop.remote.unbundle(stream, ['force'], 'push') except error.BundleValueError, exc: raise util.Abort('missing support for %s' % exc)
def unbundle(repo, proto, heads): their_heads = decodelist(heads) try: proto.redirect() exchange.check_heads(repo, their_heads, 'preparing changes') # write bundle data to temporary file because it can be big fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-') fp = os.fdopen(fd, 'wb+') r = 0 try: proto.getfile(fp) fp.seek(0) gen = exchange.readbundle(repo.ui, fp, None) r = exchange.unbundle(repo, gen, their_heads, 'serve', proto._client()) if util.safehasattr(r, 'addpart'): # The return looks streamable, we are in the bundle2 case and # should return a stream. return streamres(r.getchunks()) return pushres(r) finally: fp.close() os.unlink(tempname) except error.BundleValueError, exc: bundler = bundle2.bundle20(repo.ui) errpart = bundler.newpart('b2x:error:unsupportedcontent') if exc.parttype is not None: errpart.addparam('parttype', exc.parttype) if exc.params: errpart.addparam('params', '\0'.join(exc.params)) return streamres(bundler.getchunks())
def _pushbundle2(pushop): """push data to the remote using bundle2 The only currently supported type of data is changegroup but this will evolve in the future.""" # Send known head to the server for race detection. capsblob = urllib.unquote(pushop.remote.capable('bundle2-exp')) caps = bundle2.decodecaps(capsblob) bundler = bundle2.bundle20(pushop.ui, caps) # create reply capability capsblob = bundle2.encodecaps(pushop.repo.bundle2caps) bundler.addpart(bundle2.bundlepart('b2x:replycaps', data=capsblob)) if not pushop.force: part = bundle2.bundlepart('B2X:CHECK:HEADS', data=iter(pushop.remoteheads)) bundler.addpart(part) extrainfo = _pushbundle2extraparts(pushop, bundler) # add the changegroup bundle cg = changegroup.getlocalbundle(pushop.repo, 'push', pushop.outgoing) cgpart = bundle2.bundlepart('B2X:CHANGEGROUP', data=cg.getchunks()) bundler.addpart(cgpart) stream = util.chunkbuffer(bundler.getchunks()) try: reply = pushop.remote.unbundle(stream, ['force'], 'push') except bundle2.UnknownPartError, exc: raise util.Abort('missing support for %s' % exc)
def writebundle(ui, cg, filename, bundletype, vfs=None): """Write a bundle file and return its filename. Existing files will not be overwritten. If no filename is specified, a temporary file is created. bz2 compression can be turned off. The bundle file will be deleted in case of errors. """ fh = None cleanup = None try: if filename: if vfs: fh = vfs.open(filename, "wb") else: fh = open(filename, "wb") else: fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg") fh = os.fdopen(fd, "wb") cleanup = filename if bundletype == "HG20": import bundle2 bundle = bundle2.bundle20(ui) part = bundle.newpart('changegroup', data=cg.getchunks()) part.addparam('version', cg.version) z = nocompress() chunkiter = bundle.getchunks() else: if cg.version != '01': raise util.Abort( _('old bundle types only supports v1 ' 'changegroups')) header, compressor = bundletypes[bundletype] fh.write(header) z = compressor() chunkiter = cg.getchunks() # parse the changegroup data, otherwise we will block # in case of sshrepo because we don't know the end of the stream # an empty chunkgroup is the end of the changegroup # a changegroup has at least 2 chunkgroups (changelog and manifest). # after that, an empty chunkgroup is the end of the changegroup for chunk in chunkiter: fh.write(z.compress(chunk)) fh.write(z.flush()) cleanup = None return filename finally: if fh is not None: fh.close() if cleanup is not None: if filename and vfs: vfs.unlink(cleanup) else: os.unlink(cleanup)
def writebundle(ui, cg, filename, bundletype, vfs=None): """Write a bundle file and return its filename. Existing files will not be overwritten. If no filename is specified, a temporary file is created. bz2 compression can be turned off. The bundle file will be deleted in case of errors. """ fh = None cleanup = None try: if filename: if vfs: fh = vfs.open(filename, "wb") else: fh = open(filename, "wb") else: fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg") fh = os.fdopen(fd, "wb") cleanup = filename if bundletype == "HG20": import bundle2 bundle = bundle2.bundle20(ui) part = bundle.newpart('changegroup', data=cg.getchunks()) part.addparam('version', cg.version) z = nocompress() chunkiter = bundle.getchunks() else: if cg.version != '01': raise util.Abort(_('old bundle types only supports v1 ' 'changegroups')) header, compressor = bundletypes[bundletype] fh.write(header) z = compressor() chunkiter = cg.getchunks() # parse the changegroup data, otherwise we will block # in case of sshrepo because we don't know the end of the stream # an empty chunkgroup is the end of the changegroup # a changegroup has at least 2 chunkgroups (changelog and manifest). # after that, an empty chunkgroup is the end of the changegroup for chunk in chunkiter: fh.write(z.compress(chunk)) fh.write(z.flush()) cleanup = None return filename finally: if fh is not None: fh.close() if cleanup is not None: if filename and vfs: vfs.unlink(cleanup) else: os.unlink(cleanup)
def getbundle(repo, source, heads=None, common=None, bundlecaps=None, **kwargs): """return a full bundle (with potentially multiple kind of parts) Could be a bundle HG10 or a bundle HG2Y depending on bundlecaps passed. For now, the bundle can contain only changegroup, but this will changes when more part type will be available for bundle2. This is different from changegroup.getchangegroup that only returns an HG10 changegroup bundle. They may eventually get reunited in the future when we have a clearer idea of the API we what to query different data. The implementation is at a very early stage and will get massive rework when the API of bundle is refined. """ # bundle10 case if bundlecaps is None or 'HG2Y' not in bundlecaps: if bundlecaps and not kwargs.get('cg', True): raise ValueError( _('request for bundle10 must include changegroup')) if kwargs: raise ValueError( _('unsupported getbundle arguments: %s') % ', '.join(sorted(kwargs.keys()))) return changegroup.getchangegroup(repo, source, heads=heads, common=common, bundlecaps=bundlecaps) # bundle20 case b2caps = {} for bcaps in bundlecaps: if bcaps.startswith('bundle2='): blob = urllib.unquote(bcaps[len('bundle2='):]) b2caps.update(bundle2.decodecaps(blob)) bundler = bundle2.bundle20(repo.ui, b2caps) for name in getbundle2partsorder: func = getbundle2partsmapping[name] kwargs['heads'] = heads kwargs['common'] = common func(bundler, repo, source, bundlecaps=bundlecaps, b2caps=b2caps, **kwargs) return util.chunkbuffer(bundler.getchunks())
def unbundle(repo, proto, heads): their_heads = decodelist(heads) try: proto.redirect() exchange.check_heads(repo, their_heads, 'preparing changes') # write bundle data to temporary file because it can be big fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-') fp = os.fdopen(fd, 'wb+') r = 0 try: proto.getfile(fp) fp.seek(0) gen = exchange.readbundle(repo.ui, fp, None) r = exchange.unbundle(repo, gen, their_heads, 'serve', proto._client()) if util.safehasattr(r, 'addpart'): # The return looks streamable, we are in the bundle2 case and # should return a stream. return streamres(r.getchunks()) return pushres(r) finally: fp.close() os.unlink(tempname) except (error.BundleValueError, util.Abort, error.PushRaced), exc: # handle non-bundle2 case first if not getattr(exc, 'duringunbundle2', False): try: raise except util.Abort: # The old code we moved used sys.stderr directly. # We did not change it to minimise code change. # This need to be moved to something proper. # Feel free to do it. sys.stderr.write("abort: %s\n" % exc) return pushres(0) except error.PushRaced: return pusherr(str(exc)) bundler = bundle2.bundle20(repo.ui) for out in getattr(exc, '_bundle2salvagedoutput', ()): bundler.addpart(out) try: raise except error.BundleValueError, exc: errpart = bundler.newpart('error:unsupportedcontent') if exc.parttype is not None: errpart.addparam('parttype', exc.parttype) if exc.params: errpart.addparam('params', '\0'.join(exc.params))
def getbundle(repo, source, heads=None, common=None, bundlecaps=None, **kwargs): """return a full bundle (with potentially multiple kind of parts) Could be a bundle HG10 or a bundle HG2Y depending on bundlecaps passed. For now, the bundle can contain only changegroup, but this will changes when more part type will be available for bundle2. This is different from changegroup.getchangegroup that only returns an HG10 changegroup bundle. They may eventually get reunited in the future when we have a clearer idea of the API we what to query different data. The implementation is at a very early stage and will get massive rework when the API of bundle is refined. """ # bundle10 case if bundlecaps is None or 'HG2Y' not in bundlecaps: if bundlecaps and not kwargs.get('cg', True): raise ValueError(_('request for bundle10 must include changegroup')) if kwargs: raise ValueError(_('unsupported getbundle arguments: %s') % ', '.join(sorted(kwargs.keys()))) return changegroup.getchangegroup(repo, source, heads=heads, common=common, bundlecaps=bundlecaps) # bundle20 case b2caps = {} for bcaps in bundlecaps: if bcaps.startswith('bundle2='): blob = urllib.unquote(bcaps[len('bundle2='):]) b2caps.update(bundle2.decodecaps(blob)) bundler = bundle2.bundle20(repo.ui, b2caps) kwargs['heads'] = heads kwargs['common'] = common for name in getbundle2partsorder: func = getbundle2partsmapping[name] func(bundler, repo, source, bundlecaps=bundlecaps, b2caps=b2caps, **kwargs) return util.chunkbuffer(bundler.getchunks())
def _pushbundle2(pushop): """push data to the remote using bundle2 The only currently supported type of data is changegroup but this will evolve in the future.""" bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote)) # create reply capability capsblob = bundle2.encodecaps(pushop.repo.bundle2caps) bundler.newpart('b2x:replycaps', data=capsblob) replyhandlers = [] for partgen in bundle2partsgenerators: ret = partgen(pushop, bundler) if callable(ret): replyhandlers.append(ret) # do not push if nothing to push if bundler.nbparts <= 1: return stream = util.chunkbuffer(bundler.getchunks()) try: reply = pushop.remote.unbundle(stream, ['force'], 'push') except error.BundleValueError, exc: raise util.Abort('missing support for %s' % exc)
os.unlink(tempname) except error.BundleValueError, exc: bundler = bundle2.bundle20(repo.ui) errpart = bundler.newpart('b2x:error:unsupportedcontent') if exc.parttype is not None: errpart.addparam('parttype', exc.parttype) if exc.params: errpart.addparam('params', '\0'.join(exc.params)) return streamres(bundler.getchunks()) except util.Abort, inst: # The old code we moved used sys.stderr directly. # We did not change it to minimise code change. # This need to be moved to something proper. # Feel free to do it. if getattr(inst, 'duringunbundle2', False): bundler = bundle2.bundle20(repo.ui) manargs = [('message', str(inst))] advargs = [] if inst.hint is not None: advargs.append(('hint', inst.hint)) bundler.addpart(bundle2.bundlepart('b2x:error:abort', manargs, advargs)) return streamres(bundler.getchunks()) else: sys.stderr.write("abort: %s\n" % inst) return pushres(0) except error.PushRaced, exc: if getattr(exc, 'duringunbundle2', False): bundler = bundle2.bundle20(repo.ui) bundler.newpart('b2x:error:pushraced', [('message', str(exc))]) return streamres(bundler.getchunks())
def unbundle(repo, proto, heads): their_heads = decodelist(heads) try: proto.redirect() exchange.check_heads(repo, their_heads, 'preparing changes') # write bundle data to temporary file because it can be big fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-') fp = os.fdopen(fd, 'wb+') r = 0 try: proto.getfile(fp) fp.seek(0) gen = exchange.readbundle(repo.ui, fp, None) r = exchange.unbundle(repo, gen, their_heads, 'serve', proto._client()) if util.safehasattr(r, 'addpart'): # The return looks streamable, we are in the bundle2 case and # should return a stream. return streamres(r.getchunks()) return pushres(r) finally: fp.close() os.unlink(tempname) except (error.BundleValueError, util.Abort, error.PushRaced) as exc: # handle non-bundle2 case first if not getattr(exc, 'duringunbundle2', False): try: raise except util.Abort: # The old code we moved used sys.stderr directly. # We did not change it to minimise code change. # This need to be moved to something proper. # Feel free to do it. sys.stderr.write("abort: %s\n" % exc) return pushres(0) except error.PushRaced: return pusherr(str(exc)) bundler = bundle2.bundle20(repo.ui) for out in getattr(exc, '_bundle2salvagedoutput', ()): bundler.addpart(out) try: try: raise except error.PushkeyFailed as exc: # check client caps remotecaps = getattr(exc, '_replycaps', None) if (remotecaps is not None and 'pushkey' not in remotecaps.get('error', ())): # no support remote side, fallback to Abort handler. raise part = bundler.newpart('error:pushkey') part.addparam('in-reply-to', exc.partid) if exc.namespace is not None: part.addparam('namespace', exc.namespace, mandatory=False) if exc.key is not None: part.addparam('key', exc.key, mandatory=False) if exc.new is not None: part.addparam('new', exc.new, mandatory=False) if exc.old is not None: part.addparam('old', exc.old, mandatory=False) if exc.ret is not None: part.addparam('ret', exc.ret, mandatory=False) except error.BundleValueError as exc: errpart = bundler.newpart('error:unsupportedcontent') if exc.parttype is not None: errpart.addparam('parttype', exc.parttype) if exc.params: errpart.addparam('params', '\0'.join(exc.params)) except util.Abort as exc: manargs = [('message', str(exc))] advargs = [] if exc.hint is not None: advargs.append(('hint', exc.hint)) bundler.addpart(bundle2.bundlepart('error:abort', manargs, advargs)) except error.PushRaced as exc: bundler.newpart('error:pushraced', [('message', str(exc))]) return streamres(bundler.getchunks())
def _pushsyncphase(pushop): """synchronise phase information locally and remotely""" unfi = pushop.repo.unfiltered() cheads = pushop.commonheads # even when we don't push, exchanging phase data is useful remotephases = pushop.remote.listkeys('phases') if (pushop.ui.configbool('ui', '_usedassubrepo', False) and remotephases # server supports phases and pushop.ret is None # nothing was pushed and remotephases.get('publishing', False)): # When: # - this is a subrepo push # - and remote support phase # - and no changeset was pushed # - and remote is publishing # We may be in issue 3871 case! # We drop the possible phase synchronisation done by # courtesy to publish changesets possibly locally draft # on the remote. remotephases = {'publishing': 'True'} if not remotephases: # old server or public only reply from non-publishing _localphasemove(pushop, cheads) # don't push any phase data as there is nothing to push else: ana = phases.analyzeremotephases(pushop.repo, cheads, remotephases) pheads, droots = ana ### Apply remote phase on local if remotephases.get('publishing', False): _localphasemove(pushop, cheads) else: # publish = False _localphasemove(pushop, pheads) _localphasemove(pushop, cheads, phases.draft) ### Apply local phase on remote # Get the list of all revs draft on remote by public here. # XXX Beware that revset break if droots is not strictly # XXX root we may want to ensure it is but it is costly outdated = unfi.set('heads((%ln::%ln) and public())', droots, cheads) b2caps = bundle2.bundle2caps(pushop.remote) if 'b2x:pushkey' in b2caps: # server supports bundle2, let's do a batched push through it # # This will eventually be unified with the changesets bundle2 push bundler = bundle2.bundle20(pushop.ui, b2caps) capsblob = bundle2.encodecaps(pushop.repo.bundle2caps) bundler.newpart('b2x:replycaps', data=capsblob) part2node = [] enc = pushkey.encode for newremotehead in outdated: part = bundler.newpart('b2x:pushkey') part.addparam('namespace', enc('phases')) part.addparam('key', enc(newremotehead.hex())) part.addparam('old', enc(str(phases.draft))) part.addparam('new', enc(str(phases.public))) part2node.append((part.id, newremotehead)) stream = util.chunkbuffer(bundler.getchunks()) try: reply = pushop.remote.unbundle(stream, ['force'], 'push') op = bundle2.processbundle(pushop.repo, reply) except error.BundleValueError, exc: raise util.Abort('missing support for %s' % exc) for partid, node in part2node: partrep = op.records.getreplies(partid) results = partrep['pushkey'] assert len(results) <= 1 msg = None if not results: msg = _('server ignored update of %s to public!\n') % node elif not int(results[0]['return']): msg = _('updating %s to public failed!\n') % node if msg is not None: pushop.ui.warn(msg) else:
def unbundle(repo, proto, heads): their_heads = decodelist(heads) try: proto.redirect() exchange.check_heads(repo, their_heads, "preparing changes") # write bundle data to temporary file because it can be big fd, tempname = tempfile.mkstemp(prefix="hg-unbundle-") fp = os.fdopen(fd, "wb+") r = 0 try: proto.getfile(fp) fp.seek(0) gen = exchange.readbundle(repo.ui, fp, None) r = exchange.unbundle(repo, gen, their_heads, "serve", proto._client()) if util.safehasattr(r, "addpart"): # The return looks streamable, we are in the bundle2 case and # should return a stream. return streamres(r.getchunks()) return pushres(r) finally: fp.close() os.unlink(tempname) except (error.BundleValueError, util.Abort, error.PushRaced) as exc: # handle non-bundle2 case first if not getattr(exc, "duringunbundle2", False): try: raise except util.Abort: # The old code we moved used sys.stderr directly. # We did not change it to minimise code change. # This need to be moved to something proper. # Feel free to do it. sys.stderr.write("abort: %s\n" % exc) return pushres(0) except error.PushRaced: return pusherr(str(exc)) bundler = bundle2.bundle20(repo.ui) for out in getattr(exc, "_bundle2salvagedoutput", ()): bundler.addpart(out) try: try: raise except error.PushkeyFailed as exc: # check client caps remotecaps = getattr(exc, "_replycaps", None) if remotecaps is not None and "pushkey" not in remotecaps.get("error", ()): # no support remote side, fallback to Abort handler. raise part = bundler.newpart("error:pushkey") part.addparam("in-reply-to", exc.partid) if exc.namespace is not None: part.addparam("namespace", exc.namespace, mandatory=False) if exc.key is not None: part.addparam("key", exc.key, mandatory=False) if exc.new is not None: part.addparam("new", exc.new, mandatory=False) if exc.old is not None: part.addparam("old", exc.old, mandatory=False) if exc.ret is not None: part.addparam("ret", exc.ret, mandatory=False) except error.BundleValueError as exc: errpart = bundler.newpart("error:unsupportedcontent") if exc.parttype is not None: errpart.addparam("parttype", exc.parttype) if exc.params: errpart.addparam("params", "\0".join(exc.params)) except util.Abort as exc: manargs = [("message", str(exc))] advargs = [] if exc.hint is not None: advargs.append(("hint", exc.hint)) bundler.addpart(bundle2.bundlepart("error:abort", manargs, advargs)) except error.PushRaced as exc: bundler.newpart("error:pushraced", [("message", str(exc))]) return streamres(bundler.getchunks())