def getbundle(repo, store, heads, branch_names):
if isinstance(repo, bundlerepo):
bundle = repo._unbundler
else:
common = findcommon(repo, store, store.heads(branch_names))
logging.info('common: %s', common)
bundle = None
got_partial = False
if not common:
if not store._has_metadata and not store._graft:
manifest = Git.config('cinnabar.clone')
if manifest is None and repo.capable('cinnabarclone'):
manifest = repo._call('cinnabarclone')
if manifest:
got_partial = do_cinnabarclone(repo, manifest, store)
if not got_partial:
if check_enabled('cinnabarclone'):
raise Exception('cinnabarclone failed.')
logging.warn('Falling back to normal clone.')
if not got_partial and repo.capable('clonebundles'):
bundle = get_clonebundle(repo)
got_partial = bool(bundle)
if not got_partial and check_enabled('clonebundles'):
raise Exception('clonebundles failed.')
if bundle:
bundle = unbundler(bundle)
# Manual move semantics
apply_bundle = BundleApplier(bundle)
del bundle
apply_bundle(store)
if not changegroup:
BundleHelper.close()
if got_partial:
# Eliminate the heads that we got from the clonebundle or
# cinnabarclone.
heads = [h for h in heads if not store.changeset_ref(h)]
if not heads:
return
common = findcommon(repo, store, store.heads(branch_names))
logging.info('common: %s', common)
kwargs = {}
if unbundle20 and repo.capable('bundle2'):
bundle2caps = {
'HG20': (),
'changegroup': ('01', '02'),
}
kwargs['bundlecaps'] = set((
'HG20', 'bundle2=%s' % urllib.quote(encodecaps(bundle2caps))))
bundle = repo.getbundle('bundle', heads=[unhexlify(h) for h in heads],
common=[unhexlify(h) for h in common],
**kwargs)
bundle = unbundler(bundle)
# Manual move semantics
apply_bundle = BundleApplier(bundle)
del bundle
apply_bundle(store)
def create_hg_metadata(self, commit, parents):
if check_enabled('bundle'):
real_changeset = self.changeset(self.hg_changeset(commit))
manifest, changeset_files = self.create_hg_manifest(commit, parents)
commit_data = GitCommit(commit)
if manifest.node == NULL_NODE_ID:
manifest.node = manifest.sha1
if check_enabled('bundle'):
if real_changeset and (manifest.node !=
real_changeset.manifest):
for path, created, real in sorted_merge(
manifest,
self.manifest(real_changeset.manifest),
key=lambda i: i.path,
non_key=lambda i: i):
if bytes(created) != bytes(real):
logging.error('%r != %r', bytes(created),
bytes(real))
self._pushed.add(manifest.node)
self.store_manifest(manifest)
self._manifest_git_tree[manifest.node] = commit_data.tree
changeset = Changeset.from_git_commit(commit_data)
changeset.parents = tuple(self.hg_changeset(p) for p in parents)
changeset.manifest = manifest.node
changeset.files = changeset_files
if parents:
parent_changeset = self.changeset(changeset.parent1)
if parent_changeset.branch:
changeset.branch = parent_changeset.branch
if self._graft is True and parents and changeset.body[-1:] == b'\n':
parent_commit = GitCommit(parents[0])
if (parent_commit.body[-1:] == b'\n'
and parent_commit.body[-2] == parent_changeset.body[-1]):
self._graft = 'true'
if self._graft == 'true' and changeset.body[-1:] == b'\n':
changeset.body = changeset.body[:-1]
changeset.node = changeset.sha1
self._pushed.add(changeset.node)
self.store_changeset(changeset, commit_data)
if check_enabled('bundle') and real_changeset:
error = False
for k in ('files', 'manifest'):
if getattr(real_changeset, k, []) != getattr(changeset, k, []):
logging.error('(%s) %r != %r', k,
getattr(real_changeset, k, None),
getattr(changeset, k, None))
error = True
if error:
raise Exception('Changeset mismatch')
def create_hg_metadata(self, commit, parents):
if check_enabled('bundle'):
real_changeset = self.changeset(self.hg_changeset(commit))
manifest, changeset_files = self.create_hg_manifest(commit, parents)
commit_data = GitCommit(commit)
if manifest.node == NULL_NODE_ID:
manifest.node = manifest.sha1
if check_enabled('bundle'):
if real_changeset and (
manifest.node != real_changeset.manifest):
for path, created, real in sorted_merge(
manifest._lines,
self.manifest(real_changeset.manifest)._lines,
key=lambda i: i.name, non_key=lambda i: i):
if str(created) != str(real):
logging.error('%r != %r', str(created), str(real))
self._pushed.add(manifest.node)
self.store_manifest(manifest)
self._manifest_git_tree[manifest.node] = commit_data.tree
changeset = Changeset.from_git_commit(commit_data)
changeset.parents = tuple(self.hg_changeset(p) for p in parents)
changeset.manifest = manifest.node
changeset.files = changeset_files
if parents:
parent_changeset = self.changeset(changeset.parent1)
if parent_changeset.branch:
changeset.branch = parent_changeset.branch
if self._graft is True and parents and changeset.body[-1] == '\n':
parent_commit = GitCommit(parents[0])
if (parent_commit.body[-1] == '\n' and
parent_commit.body[-2] == parent_changeset.body[-1]):
self._graft = 'true'
if self._graft == 'true' and changeset.body[-1] == '\n':
changeset.body = changeset.body[:-1]
changeset.node = changeset.sha1
self._pushed.add(changeset.node)
self.store_changeset(changeset, commit_data)
if check_enabled('bundle') and real_changeset:
error = False
for k in ('files', 'manifest'):
if getattr(real_changeset, k, []) != getattr(changeset, k, []):
logging.error('(%s) %r != %r', k,
getattr(real_changeset, k, None),
getattr(changeset, k, None))
error = True
if error:
raise Exception('Changeset mismatch')
def get_repo(remote):
if not changegroup or experiment('wire'):
if not changegroup and not check_enabled('no-mercurial'):
logging.warning('Mercurial libraries not found. Falling back to '
'native access.')
logging.warning(
'Native access to mercurial repositories is experimental!')
stream = HgRepoHelper.connect(remote.url)
if stream:
return bundlerepo(remote.url, stream)
return HelperRepo(remote.url)
if remote.parsed_url.scheme == 'file':
path = remote.parsed_url.path
if sys.platform == 'win32':
# TODO: This probably needs more thought.
path = path.lstrip('/')
if not os.path.isdir(path):
return bundlerepo(path)
ui = get_ui()
if changegroup and remote.parsed_url.scheme == 'file':
repo = localpeer(ui, path)
else:
try:
repo = hg.peer(ui, {}, remote.url)
except (error.RepoError, urllib2.HTTPError, IOError):
return bundlerepo(remote.url, url.open(ui, remote.url))
assert repo.capable('getbundle')
return repo
def iter_initialized(get_missing, iterable, init=None):
previous = None
always_check = check_enabled('nodeid')
for instance in iterable:
check = always_check
if instance.delta_node != NULL_NODE_ID:
if not previous or instance.delta_node != previous.node:
previous = get_missing(instance.delta_node)
check = True
if init:
instance = init(instance, previous)
else:
instance.init(previous)
elif init:
instance = init(instance)
else:
instance.init(())
if check and instance.node != instance.sha1:
raise Exception(
'sha1 mismatch for node %s with parents %s %s and '
'previous %s' %
(instance.node, instance.parent1, instance.parent2,
instance.delta_node)
)
yield instance
previous = instance
def _get_repo(remote):
if not changegroup or experiment('wire'):
if not changegroup and not check_enabled('no-mercurial'):
logging.warning('Mercurial libraries not found. Falling back to '
'experimental native access.')
stream = HgRepoHelper.connect(remote.url)
if stream:
return bundlerepo(remote.url, stream)
return HelperRepo(remote.url)
if remote.parsed_url.scheme == b'file':
# Make file://c:/... paths work by taking the netloc
path = remote.parsed_url.netloc + remote.parsed_url.path
if sys.platform == 'win32':
# TODO: This probably needs more thought.
path = path.lstrip(b'/')
if not os.path.isdir(path):
return bundlerepo(path)
ui = get_ui()
if changegroup and remote.parsed_url.scheme == b'file':
repo = localpeer(ui, path)
else:
try:
repo = hg.peer(ui, {}, remote.url)
except (error.RepoError, HTTPError, IOError):
if remote.parsed_url.scheme in ('http', 'https'):
return bundlerepo(remote.url, HTTPReader(remote.url))
raise
assert repo.capable(b'getbundle')
return repo
def get_repo(remote):
if not changegroup or experiment('wire'):
if not changegroup and not check_enabled('no-mercurial'):
logging.warning('Mercurial libraries not found. Falling back to '
'native access.')
logging.warning(
'Native access to mercurial repositories is experimental!')
stream = HgRepoHelper.connect(remote.url)
if stream:
return bundlerepo(remote.url, stream)
return HelperRepo(remote.url)
if remote.parsed_url.scheme == 'file':
# Make file://c:/... paths work by taking the netloc
path = remote.parsed_url.netloc + remote.parsed_url.path
if sys.platform == 'win32':
# TODO: This probably needs more thought.
path = path.lstrip('/')
if not os.path.isdir(path):
return bundlerepo(path)
ui = get_ui()
if changegroup and remote.parsed_url.scheme == 'file':
repo = localpeer(ui, path)
else:
try:
repo = hg.peer(ui, {}, remote.url)
except (error.RepoError, urllib2.HTTPError, IOError):
return bundlerepo(remote.url, HTTPReader(remote.url))
assert repo.capable('getbundle')
return repo
def get_clonebundle(repo):
url = Git.config('cinnabar.clonebundle')
if not url:
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
return None
bundles = repo._call('clonebundles')
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
url = entries[0].get('URL')
if not url:
return None
sys.stderr.write('Getting clone bundle from %s\n' % url)
return unbundle_fh(HTTPReader(url), url)
def iter_initialized(get_missing, iterable):
previous = None
always_check = check_enabled('nodeid')
for instance in iterable:
check = always_check
if instance.delta_node != NULL_NODE_ID:
if previous and instance.delta_node == previous.node:
instance.init(previous)
else:
instance.init(get_missing(instance.delta_node))
check = True
else:
instance.init(())
if check and instance.node != instance.sha1:
raise Exception(
'sha1 mismatch for node %s with parents %s %s and '
'previous %s' %
(instance.node, instance.parent1, instance.parent2,
instance.delta_node)
)
yield instance
previous = instance
def iter_initialized(get_missing, iterable, init=None):
previous = None
check = check_enabled('nodeid')
for instance in iterable:
if instance.delta_node != NULL_NODE_ID:
if not previous or instance.delta_node != previous.node:
previous = get_missing(instance.delta_node)
if init:
instance = init(instance, previous)
else:
instance.init(previous)
elif init:
instance = init(instance)
else:
instance.init(())
if check and instance.node != instance.sha1:
raise Exception(
'sha1 mismatch for node %s with parents %s %s and '
'previous %s' %
(instance.node, instance.parent1, instance.parent2,
instance.delta_node)
)
yield instance
previous = instance
def getbundle(repo, store, heads, branch_names):
if isinstance(repo, bundlerepo):
bundle = repo._unbundler
else:
common = findcommon(repo, store, store.heads(branch_names))
logging.info('common: %s', common)
bundle = None
got_partial = False
if not common:
if not store._has_metadata:
manifest = Git.config('cinnabar.clone', remote=repo.remote)
limit_schemes = False
if manifest is None and repo.capable(b'cinnabarclone'):
# If no cinnabar.clone config was given, but a
# cinnabar.clonebundle config was, act as if an empty
# cinnabar.clone config had been given, and proceed with
# the mercurial clonebundle.
if not Git.config('cinnabar.clonebundle',
remote=repo.remote):
manifest = repo._call(b'cinnabarclone')
limit_schemes = True
if manifest:
got_partial = do_cinnabarclone(repo, manifest, store,
limit_schemes)
if not got_partial:
if check_enabled('cinnabarclone'):
raise Exception('cinnabarclone failed.')
logging.warn('Falling back to normal clone.')
if not got_partial and repo.capable(b'clonebundles'):
bundle = get_clonebundle(repo)
got_partial = bool(bundle)
if not got_partial and check_enabled('clonebundles'):
raise Exception('clonebundles failed.')
if bundle:
bundle = unbundler(bundle)
# Manual move semantics
apply_bundle = BundleApplier(bundle)
del bundle
apply_bundle(store)
if not changegroup:
BundleHelper.close()
if got_partial:
# Eliminate the heads that we got from the clonebundle or
# cinnabarclone.
heads = [h for h in heads if not store.changeset_ref(h)]
if not heads:
return
common = findcommon(repo, store, store.heads(branch_names))
logging.info('common: %s', common)
kwargs = {}
if unbundle20 and repo.capable(b'bundle2'):
bundle2caps = {
b'HG20': (),
b'changegroup': (b'01', b'02'),
}
kwargs['bundlecaps'] = set(
(b'HG20', b'bundle2=%s' %
quote_from_bytes(encodecaps(bundle2caps)).encode('ascii')))
bundle = repo.getbundle(b'bundle',
heads=[unhexlify(h) for h in heads],
common=[unhexlify(h) for h in common],
**kwargs)
bundle = unbundler(bundle)
# Manual move semantics
apply_bundle = BundleApplier(bundle)
del bundle
apply_bundle(store)
from collections import (
defaultdict,
deque,
)
from .bundle import (
create_bundle,
encodecaps,
decodecaps,
)
from .changegroup import (
RawRevChunk01,
RawRevChunk02,
)
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
# Old versions of mercurial use an old version of socketutil that tries to
# assign a local PROTOCOL_SSLv2, copying it from the ssl module, without
# ever using it. It shouldn't hurt to set it here.
if not hasattr(ssl, 'PROTOCOL_SSLv2'):
ssl.PROTOCOL_SSLv2 = 0
if not hasattr(ssl, 'PROTOCOL_SSLv3'):
ssl.PROTOCOL_SSLv3 = 1
from mercurial import (
changegroup,
error,
hg,
ui,
url,
def bundle_data(store, commits):
manifests = OrderedDict()
files = defaultdict(list)
for node, parents in progress_iter('Bundling {} changesets', commits):
if len(parents) > 2:
raise Exception(
'Pushing octopus merges to mercurial is not supported')
changeset_data = store.read_changeset_data(node)
is_new = changeset_data is None or check_enabled('bundle')
if is_new:
store.create_hg_metadata(node, parents)
hg_changeset = store._changeset(node, include_parents=True)
if is_new:
store.add_head(hg_changeset.node, hg_changeset.parent1,
hg_changeset.parent2)
yield hg_changeset
manifest = hg_changeset.manifest
if manifest not in manifests and manifest != NULL_NODE_ID:
if manifest not in (store.changeset(p).manifest
for p in hg_changeset.parents):
manifests[manifest] = hg_changeset.node
yield None
for manifest, changeset in progress_iter('Bundling {} manifests',
manifests.iteritems()):
hg_manifest = store.manifest(manifest, include_parents=True)
hg_manifest.changeset = changeset
yield hg_manifest
manifest_ref = store.manifest_ref(manifest)
parents = tuple(store.manifest_ref(p) for p in hg_manifest.parents)
changes = get_changes(manifest_ref, parents)
for path, hg_file, hg_fileparents in changes:
if hg_file != NULL_NODE_ID:
files[store.manifest_path(path)].append(
(hg_file, hg_fileparents, changeset, parents))
yield None
def iter_files(files):
for path in sorted(files):
yield path
nodes = set()
for node, parents, changeset, mn_parents in files[path]:
if node in nodes:
continue
nodes.add(node)
file = store.file(node, parents, mn_parents, path)
file.changeset = changeset
assert file.node == file.sha1
yield file
yield None
class Filt(object):
def __init__(self):
self._previous = None
def __call__(self, chunk):
ret = self._previous and chunk is not None
self._previous = chunk
return ret
for chunk in progress_iter('Bundling {} files', iter_files(files), Filt()):
yield chunk
yield None
def bundle_data(store, commits):
manifests = OrderedDict()
files = defaultdict(list)
for node, parents in progress_iter('Bundling {} changesets', commits):
if len(parents) > 2:
raise Exception(
'Pushing octopus merges to mercurial is not supported')
changeset_data = store.read_changeset_data(node)
is_new = changeset_data is None or check_enabled('bundle')
if is_new:
store.create_hg_metadata(node, parents)
hg_changeset = store._changeset(node, include_parents=True)
if is_new:
store.add_head(hg_changeset.node, hg_changeset.parent1,
hg_changeset.parent2)
yield hg_changeset
manifest = hg_changeset.manifest
if manifest not in manifests and manifest != NULL_NODE_ID:
if manifest not in (store.changeset(p).manifest
for p in hg_changeset.parents):
manifests[manifest] = hg_changeset.node
yield None
for manifest, changeset in progress_iter('Bundling {} manifests',
manifests.iteritems()):
hg_manifest = store.manifest(manifest, include_parents=True)
hg_manifest.changeset = changeset
yield hg_manifest
manifest_ref = store.manifest_ref(manifest)
parents = tuple(store.manifest_ref(p) for p in hg_manifest.parents)
changes = get_changes(manifest_ref, parents)
for path, hg_file, hg_fileparents in changes:
if hg_file != NULL_NODE_ID:
files[store.manifest_path(path)].append(
(hg_file, hg_fileparents, changeset, parents))
yield None
def iter_files(files):
count_chunks = 0
for count_names, path in enumerate(sorted(files), 1):
yield (count_chunks, count_names), path
nodes = set()
for node, parents, changeset, mn_parents in files[path]:
if node in nodes:
continue
count_chunks += 1
nodes.add(node)
file = store.file(node, parents, mn_parents, path)
file.changeset = changeset
assert file.node == file.sha1
yield (count_chunks, count_names), file
yield (count_chunks, count_names), None
for chunk in progress_enum('Bundling {} revisions of {} files',
iter_files(files)):
yield chunk
yield None
def get_clonebundle_url(repo):
bundles = repo._call(b'clonebundles')
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
parseclonebundlesmanifest = False
if parseclonebundlesmanifest:
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
return entries[0].get(b'URL')
supported_bundles = (b'v1', b'v2')
supported_compressions = tuple(k for k, v in (
(b'none', b'UN'),
(b'gzip', b'GZ'),
(b'bzip2', b'BZ'),
(b'zstd', b'ZS'),
) if HgRepoHelper.supports((b'compression', v)))
has_sni = getattr(ssl, 'HAS_SNI', False)
logger = logging.getLogger('clonebundle')
for line in bundles.splitlines():
attrs = line.split()
if not attrs:
continue
url = attrs.pop(0)
logger.debug(url)
attrs = {
unquote_to_bytes(k): unquote_to_bytes(v)
for k, _, v in (a.partition(b'=') for a in attrs)
}
logger.debug(attrs)
if b'REQUIRESNI' in attrs and not has_sni:
logger.debug('Skip because of REQUIRESNI, but SNI unsupported')
continue
spec = attrs.get(b'BUNDLESPEC')
if not spec:
logger.debug('Skip because missing BUNDLESPEC')
continue
typ, _, params = spec.partition(b';')
compression, _, version = typ.partition(b'-')
if compression not in supported_compressions:
logger.debug('Skip because unsupported compression (%s)',
compression)
continue
if version not in supported_bundles:
logger.debug('Skip because unsupported bundle type (%s)', version)
continue
params_dict = {}
for p in params.split(b':'):
k, _, v = p.partition(b'=')
params_dict[k] = v
if 'stream' in params_dict:
logger.debug('Skip because stream bundles are not supported')
continue
return url
def bundle_data(store, commits):
manifests = OrderedDict()
files = defaultdict(list)
for node, parents in progress_iter('Bundling {} changesets', commits):
if len(parents) > 2:
raise Exception(
'Pushing octopus merges to mercurial is not supported')
changeset_data = store.read_changeset_data(node)
is_new = changeset_data is None or check_enabled('bundle')
if is_new:
store.create_hg_metadata(node, parents)
hg_changeset = store._changeset(node, include_parents=True)
if is_new:
store.add_head(hg_changeset.node, hg_changeset.parent1,
hg_changeset.parent2)
yield hg_changeset
manifest = hg_changeset.manifest
if manifest not in manifests and manifest != NULL_NODE_ID:
if manifest not in (store.changeset(p).manifest
for p in hg_changeset.parents):
manifests[manifest] = hg_changeset.node
yield None
for manifest, changeset in progress_iter('Bundling {} manifests',
iteritems(manifests)):
hg_manifest = store.manifest(manifest, include_parents=True)
hg_manifest.changeset = changeset
yield hg_manifest
manifest_ref = store.manifest_ref(manifest)
parents = tuple(store.manifest_ref(p) for p in hg_manifest.parents)
changes = get_changes(manifest_ref, parents)
for path, hg_file, hg_fileparents in changes:
if hg_file != NULL_NODE_ID:
files[store.manifest_path(path)].append(
(hg_file, hg_fileparents, changeset, parents))
yield None
def iter_files(files):
count_chunks = 0
for count_names, path in enumerate(sorted(files), 1):
yield (count_chunks, count_names), path
nodes = set()
for node, parents, changeset, mn_parents in files[path]:
if node in nodes:
continue
count_chunks += 1
nodes.add(node)
file = store.file(node, parents, mn_parents, path)
file.changeset = changeset
assert file.node == file.sha1
yield (count_chunks, count_names), file
yield (count_chunks, count_names), None
for chunk in progress_enum('Bundling {} revisions of {} files',
iter_files(files)):
yield chunk
yield None
def get_clonebundle(repo):
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
return None
bundles = repo._call('clonebundles')
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
url = entries[0].get('URL')
if not url:
return None
sys.stderr.write('Getting clone bundle from %s\n' % url)
class Getter(object):
def __init__(self, url):
self.fh = urllib2.urlopen(url)
self.url = url
try:
self.length = int(self.fh.headers['content-length'])
except (ValueError, KeyError):
self.length = None
self.offset = 0
def read(self, size):
try:
result = self.fh.read(size)
except socket.error:
result = ''
# When self.length is None, self.offset < self.length is always
# false.
if not result and self.offset < self.length:
# Processing large manifests or large files can be slow.
# With highly compressed but nevertheless large bundles, this
# means it can take time to process relatively small
# (compressed) inputs: in the order of several minutes for a
# few megabytes. When that happens, SSL connections can end
# up being aborted between two large TCP receives. In that
# case, try again with an HTTP Range request if the server
# supports it.
# TODO: This is a stopgap until processing is faster.
req = urllib2.Request(self.url)
req.add_header('Range', 'bytes=%d-' % self.offset)
self.fh = urllib2.urlopen(req)
if self.fh.getcode() != 206:
return ''
range = self.fh.headers['Content-Range'].split(None, 1)
if len(range) != 2:
return ''
unit, range = range
if unit != 'bytes':
return ''
range = range.split('-', 1)
if len(range) != 2:
return ''
start, end = range
start = int(start)
if start > self.offset:
return ''
logging.getLogger('clonebundle').debug(
'Retrying from offset %d', start)
while start < self.offset:
l = len(self.fh.read(self.offset - start))
if not l:
return ''
start += l
result = self.fh.read(size)
self.offset += len(result)
return result
return unbundle_fh(Getter(url), url)
progress_enum,
progress_iter,
)
from collections import (
defaultdict,
deque,
)
from .bundle import create_bundle
from .changegroup import (
RawRevChunk01,
RawRevChunk02,
)
from cStringIO import StringIO
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
# Old versions of mercurial use an old version of socketutil that tries to
# assign a local PROTOCOL_SSLv2, copying it from the ssl module, without
# ever using it. It shouldn't hurt to set it here.
import ssl
if not hasattr(ssl, 'PROTOCOL_SSLv2'):
ssl.PROTOCOL_SSLv2 = 0
if not hasattr(ssl, 'PROTOCOL_SSLv3'):
ssl.PROTOCOL_SSLv3 = 1
from mercurial import (
changegroup,
error,
hg,
ui,
def get_clonebundle_url(repo):
bundles = repo._call('clonebundles')
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
parseclonebundlesmanifest = False
if parseclonebundlesmanifest:
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
return entries[0].get('URL')
supported_bundles = ('v1', 'v2')
supported_compressions = tuple(
k for k, v in (
('none', 'UN'),
('gzip', 'GZ'),
('bzip2', 'BZ'),
('zstd', 'ZS'),
) if HgRepoHelper.supports(('compression', v))
)
has_sni = getattr(ssl, 'HAS_SNI', False)
logger = logging.getLogger('clonebundle')
for line in bundles.splitlines():
attrs = line.split()
if not attrs:
continue
url = attrs.pop(0)
logger.debug(url)
attrs = {
urllib.unquote(k): urllib.unquote(v)
for k, _, v in (a.partition('=') for a in attrs)
}
logger.debug(attrs)
if 'REQUIRESNI' in attrs and not has_sni:
logger.debug('Skip because of REQUIRESNI, but SNI unsupported')
continue
spec = attrs.get('BUNDLESPEC')
if not spec:
logger.debug('Skip because missing BUNDLESPEC')
continue
typ, _, params = spec.partition(';')
compression, _, version = typ.partition('-')
if compression not in supported_compressions:
logger.debug('Skip because unsupported compression (%s)',
compression)
continue
if version not in supported_bundles:
logger.debug('Skip because unsupported bundle type (%s)',
version)
continue
return url
def getbundle(repo, store, heads, branch_names):
if isinstance(repo, bundlerepo):
bundle = repo._unbundler
else:
common = findcommon(repo, store, store.heads(branch_names))
logging.info('common: %s', common)
bundle = None
got_partial = False
if not common:
if not store._has_metadata and not store._graft:
manifest = Git.config('cinnabar.clone')
if not manifest and experiment('git-clone') and \
repo.capable('cinnabarclone'):
manifest = repo._call('cinnabarclone')
if manifest:
got_partial = do_cinnabarclone(repo, manifest, store)
if not got_partial:
if check_enabled('cinnabarclone'):
raise Exception('cinnabarclone failed.')
logging.warn('Falling back to normal clone.')
if not got_partial and repo.capable('clonebundles'):
bundle = get_clonebundle(repo)
got_partial = bool(bundle)
if not got_partial and check_enabled('clonebundles'):
raise Exception('clonebundles failed.')
if bundle:
bundle = unbundler(bundle)
# Manual move semantics
apply_bundle = BundleApplier(bundle)
del bundle
apply_bundle(store)
if not changegroup:
BundleHelper.close()
if got_partial:
# Eliminate the heads that we got from the clonebundle or
# cinnabarclone.
heads = [h for h in heads if not store.changeset_ref(h)]
if not heads:
return
common = findcommon(repo, store, store.heads(branch_names))
logging.info('common: %s', common)
kwargs = {}
if unbundle20 and repo.capable('bundle2'):
bundle2caps = {
'HG20': (),
'changegroup': ('01', '02'),
}
kwargs['bundlecaps'] = set((
'HG20', 'bundle2=%s' % urllib.quote(encodecaps(bundle2caps))))
bundle = repo.getbundle('bundle', heads=[unhexlify(h) for h in heads],
common=[unhexlify(h) for h in common],
**kwargs)
bundle = unbundler(bundle)
# Manual move semantics
apply_bundle = BundleApplier(bundle)
del bundle
apply_bundle(store)
def create_hg_metadata(self, commit, parents):
if check_enabled('bundle'):
real_changeset_data = self.read_changeset_data(commit)
manifest = self.create_hg_manifest(commit, parents)
commit_data = GitCommit(commit)
if manifest.node == NULL_NODE_ID:
manifest.node = manifest.sha1
if check_enabled('bundle'):
if real_changeset_data and (manifest.node !=
real_changeset_data['manifest']):
for path, created, real in sorted_merge(
manifest._lines,
self.manifest(
real_changeset_data['manifest'])._lines,
key=lambda i: i.name,
non_key=lambda i: i):
if str(created) != str(real):
logging.error('%r != %r', str(created), str(real))
self._push_manifests[manifest.node] = manifest
self.manifest_ref(manifest.node, hg2git=False, create=True)
self._manifest_git_tree[manifest.node] = commit_data.tree
extra = {}
if commit_data.author != commit_data.committer:
committer = self.hg_author_info(commit_data.committer)
extra['committer'] = '%s %d %d' % committer
if parents:
parent_changeset_data = self.read_changeset_data(parents[0])
branch = parent_changeset_data.get('extra', {}).get('branch')
if branch:
extra['branch'] = branch
changeset_data = self._changeset_data_cache[commit] = {
'files': sorted(chain(manifest.removed, manifest.modified)),
'manifest': manifest.node,
}
if extra:
changeset_data['extra'] = extra
changeset = self._changeset(commit, include_parents=True)
if self._graft is True and parents and changeset.data[-1] == '\n':
parent_cs = self._changeset(parents[0], skip_patch=True)
if 'patch' not in self._changeset_data_cache[parents[0]]:
self._graft = False
else:
patch = self._changeset_data_cache[parents[0]]['patch'][-1]
self._graft = (patch[1] == len(parent_cs.data)
and parent_cs.data[-1] == '\n')
if self._graft:
self._graft = 'true'
if self._graft == 'true' and changeset.data[-1] == '\n':
changeset.data = changeset.data[:-1]
changeset_data['patch'] = ((len(changeset.data),
len(changeset.data) + 1, ''), )
changeset_data['changeset'] = changeset.changeset = changeset.node = \
changeset.sha1
self._push_changesets[changeset.node] = changeset
# This is a horrible way to do this, but this method is not doing much
# better overall anyways.
if extra:
if 'committer' in extra:
del extra['committer']
if not extra:
del changeset_data['extra']
self._changesets[changeset.node] = PseudoString(commit)
if check_enabled('bundle') and real_changeset_data:
error = False
for k in ('files', 'manifest'):
if real_changeset_data.get(k, []) != changeset_data.get(k):
logging.error('(%s) %r != %r', k,
real_changeset_data.get(k),
changeset_data.get(k))
error = True
if error:
raise Exception('Changeset mismatch')
defaultdict,
deque,
)
from .bundle import (
create_bundle,
encodecaps,
decodecaps,
)
from .changegroup import (
RawRevChunk01,
RawRevChunk02,
)
from cStringIO import StringIO
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
# Old versions of mercurial use an old version of socketutil that tries to
# assign a local PROTOCOL_SSLv2, copying it from the ssl module, without
# ever using it. It shouldn't hurt to set it here.
if not hasattr(ssl, 'PROTOCOL_SSLv2'):
ssl.PROTOCOL_SSLv2 = 0
if not hasattr(ssl, 'PROTOCOL_SSLv3'):
ssl.PROTOCOL_SSLv3 = 1
from mercurial import (
changegroup,
error,
hg,
ui,
url,