def get_clonebundle(repo):
try:
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
return None
bundles = repo._call('clonebundles')
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
url = entries[0].get('URL')
if not url:
return None
sys.stderr.write('Getting clone bundle from %s\n' % url)
return unbundle_fh(urllib2.urlopen(url), url)
def get_clonebundle(repo):
url = Git.config('cinnabar.clonebundle')
if not url:
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
return None
bundles = repo._call('clonebundles')
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
url = entries[0].get('URL')
if not url:
return None
sys.stderr.write('Getting clone bundle from %s\n' % url)
return unbundle_fh(HTTPReader(url), url)
def get_clonebundle(repo):
try:
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
return None
bundles = repo._call('clonebundles')
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
url = entries[0].get('URL')
if not url:
return None
sys.stderr.write('Getting clone bundle from %s\n' % url)
class Getter(object):
def __init__(self, url):
self.fh = urllib2.urlopen(url)
self.offset = 0
def read(self, size):
try:
result = self.fh.read(size)
except socket.error as e:
# Processing large manifests can be slow. Especially with
# changegroup v2 and lots of consecutive manifests not being
# directly connected (such that the diff is not against the
# last one).
# With highly compressed but nevertheless large bundles, this
# means it can take time to process relatively small
# (compressed) inputs: in the order of several minutes for a
# few megabytes. When that happens, SSL connections can end
# up being aborted between two large TCP receives. In that
# case, try again with an HTTP Range request if the server
# supports it.
# TODO: This is a stopgap until manifest processing is faster.
req = urllib2.Request(url)
req.add_header('Range', 'bytes=%d-' % self.offset)
self.fh = urllib2.urlopen(req)
if self.fh.getcode() != 206:
raise e
range = self.fh.headers['Content-Range'].split(None, 1)
if len(range) != 2:
raise e
unit, range = range
if unit != 'bytes':
raise e
range = range.split('-', 1)
if len(range) != 2:
raise e
start, end = range
start = int(start)
if start > self.offset:
raise e
logging.getLogger('clonebundle').debug(
'Retrying from offset %d', start)
while start < self.offset:
start += len(self.fh.read(self.offset - start))
result = self.fh.read(size)
self.offset += len(result)
return result
return unbundle_fh(Getter(url), url)
def get_clonebundle_url(repo):
bundles = repo._call(b'clonebundles')
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
parseclonebundlesmanifest = False
if parseclonebundlesmanifest:
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
return entries[0].get(b'URL')
supported_bundles = (b'v1', b'v2')
supported_compressions = tuple(k for k, v in (
(b'none', b'UN'),
(b'gzip', b'GZ'),
(b'bzip2', b'BZ'),
(b'zstd', b'ZS'),
) if HgRepoHelper.supports((b'compression', v)))
has_sni = getattr(ssl, 'HAS_SNI', False)
logger = logging.getLogger('clonebundle')
for line in bundles.splitlines():
attrs = line.split()
if not attrs:
continue
url = attrs.pop(0)
logger.debug(url)
attrs = {
unquote_to_bytes(k): unquote_to_bytes(v)
for k, _, v in (a.partition(b'=') for a in attrs)
}
logger.debug(attrs)
if b'REQUIRESNI' in attrs and not has_sni:
logger.debug('Skip because of REQUIRESNI, but SNI unsupported')
continue
spec = attrs.get(b'BUNDLESPEC')
if not spec:
logger.debug('Skip because missing BUNDLESPEC')
continue
typ, _, params = spec.partition(b';')
compression, _, version = typ.partition(b'-')
if compression not in supported_compressions:
logger.debug('Skip because unsupported compression (%s)',
compression)
continue
if version not in supported_bundles:
logger.debug('Skip because unsupported bundle type (%s)', version)
continue
params_dict = {}
for p in params.split(b':'):
k, _, v = p.partition(b'=')
params_dict[k] = v
if 'stream' in params_dict:
logger.debug('Skip because stream bundles are not supported')
continue
return url
def get_clonebundle(repo):
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
return None
bundles = repo._call('clonebundles')
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
url = entries[0].get('URL')
if not url:
return None
sys.stderr.write('Getting clone bundle from %s\n' % url)
class Getter(object):
def __init__(self, url):
self.fh = urllib2.urlopen(url)
self.url = url
try:
self.length = int(self.fh.headers['content-length'])
except (ValueError, KeyError):
self.length = None
self.offset = 0
def read(self, size):
try:
result = self.fh.read(size)
except socket.error:
result = ''
# When self.length is None, self.offset < self.length is always
# false.
if not result and self.offset < self.length:
# Processing large manifests or large files can be slow.
# With highly compressed but nevertheless large bundles, this
# means it can take time to process relatively small
# (compressed) inputs: in the order of several minutes for a
# few megabytes. When that happens, SSL connections can end
# up being aborted between two large TCP receives. In that
# case, try again with an HTTP Range request if the server
# supports it.
# TODO: This is a stopgap until processing is faster.
req = urllib2.Request(self.url)
req.add_header('Range', 'bytes=%d-' % self.offset)
self.fh = urllib2.urlopen(req)
if self.fh.getcode() != 206:
return ''
range = self.fh.headers['Content-Range'].split(None, 1)
if len(range) != 2:
return ''
unit, range = range
if unit != 'bytes':
return ''
range = range.split('-', 1)
if len(range) != 2:
return ''
start, end = range
start = int(start)
if start > self.offset:
return ''
logging.getLogger('clonebundle').debug(
'Retrying from offset %d', start)
while start < self.offset:
l = len(self.fh.read(self.offset - start))
if not l:
return ''
start += l
result = self.fh.read(size)
self.offset += len(result)
return result
return unbundle_fh(Getter(url), url)
def get_clonebundle_url(repo):
bundles = repo._call('clonebundles')
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
parseclonebundlesmanifest = False
if parseclonebundlesmanifest:
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
return entries[0].get('URL')
supported_bundles = ('v1', 'v2')
supported_compressions = tuple(
k for k, v in (
('none', 'UN'),
('gzip', 'GZ'),
('bzip2', 'BZ'),
('zstd', 'ZS'),
) if HgRepoHelper.supports(('compression', v))
)
has_sni = getattr(ssl, 'HAS_SNI', False)
logger = logging.getLogger('clonebundle')
for line in bundles.splitlines():
attrs = line.split()
if not attrs:
continue
url = attrs.pop(0)
logger.debug(url)
attrs = {
urllib.unquote(k): urllib.unquote(v)
for k, _, v in (a.partition('=') for a in attrs)
}
logger.debug(attrs)
if 'REQUIRESNI' in attrs and not has_sni:
logger.debug('Skip because of REQUIRESNI, but SNI unsupported')
continue
spec = attrs.get('BUNDLESPEC')
if not spec:
logger.debug('Skip because missing BUNDLESPEC')
continue
typ, _, params = spec.partition(';')
compression, _, version = typ.partition('-')
if compression not in supported_compressions:
logger.debug('Skip because unsupported compression (%s)',
compression)
continue
if version not in supported_bundles:
logger.debug('Skip because unsupported bundle type (%s)',
version)
continue
return url