def pull(self, url):
"""
Tries to pull changes from external location.
"""
url = self._get_url(url)
other = peer(self._repo, {}, url)
try:
# hg 3.2 moved push / pull to exchange module
from mercurial import exchange
exchange.pull(self._repo, other, heads=None, force=None)
except ImportError:
self._repo.pull(other, heads=None, force=None)
except Abort as err:
# Propagate error but with vcs's type
raise RepositoryError(str(err))
def pull(self, url):
"""
Tries to pull changes from external location.
"""
url = self._get_url(url)
other = peer(self._repo, {}, url)
try:
# hg 3.2 moved push / pull to exchange module
from mercurial import exchange
exchange.pull(self._repo, other, heads=None, force=None)
except ImportError:
self._repo.pull(other, heads=None, force=None)
except Abort, err:
# Propagate error but with vcs's type
raise RepositoryError(str(err))
def _widen(ui, repo, remote, commoninc, newincludes, newexcludes):
newmatch = narrowspec.match(repo.root, newincludes, newexcludes)
# TODO(martinvonz): Get expansion working with widening/narrowing.
if narrowspec.needsexpansion(newincludes):
raise error.Abort('Expansion not yet supported on pull')
def pullbundle2extraprepare_widen(orig, pullop, kwargs):
orig(pullop, kwargs)
# The old{in,ex}cludepats have already been set by orig()
kwargs['includepats'] = newincludes
kwargs['excludepats'] = newexcludes
wrappedextraprepare = extensions.wrappedfunction(
exchange, '_pullbundle2extraprepare', pullbundle2extraprepare_widen)
# define a function that narrowbundle2 can call after creating the
# backup bundle, but before applying the bundle from the server
def setnewnarrowpats():
repo.setnarrowpats(newincludes, newexcludes)
repo.setnewnarrowpats = setnewnarrowpats
ds = repo.dirstate
p1, p2 = ds.p1(), ds.p2()
with ds.parentchange():
ds.setparents(node.nullid, node.nullid)
common = commoninc[0]
with wrappedextraprepare:
exchange.pull(repo, remote, heads=common)
with ds.parentchange():
ds.setparents(p1, p2)
actions = {k: [] for k in 'a am f g cd dc r dm dg m e k p pr'.split()}
addgaction = actions['g'].append
mf = repo['.'].manifest().matches(newmatch)
for f, fn in mf.iteritems():
if f not in repo.dirstate:
addgaction(
(f, (mf.flags(f), False), "add from widened narrow clone"))
merge.applyupdates(repo,
actions,
wctx=repo[None],
mctx=repo['.'],
overwrite=False)
merge.recordupdates(repo, actions, branchmerge=False)
def pull(repo, remote):
"""41421bd9c42e dropped localrepo.pull"""
repo.invalidate()
try:
return repo.pull(remote)
except AttributeError:
return exchange.pull(repo, remote).cgresult
def aggregate_once(ui, repo):
oldlen = len(repo)
pullcount = 0
for name, url in ui.configitems('paths'):
remote = hg.peer(repo, {}, url)
try:
exchange.pull(repo, remote)
pullcount += 1
except NoIncomingError:
pass
newlen = len(repo)
delta = newlen - oldlen
if delta:
ui.status('aggregated %d changesets from %d repos\n' % (delta, pullcount))
else:
ui.status('no changesets aggregated\n')
def setbranch(self, branch, pbranches):
if not self.clonebranches:
return
setbranch = (branch != self.lastbranch)
self.lastbranch = branch
if not branch:
branch = 'default'
pbranches = [(b[0], b[1] and b[1] or 'default') for b in pbranches]
if pbranches:
pbranch = pbranches[0][1]
else:
pbranch = 'default'
branchpath = os.path.join(self.path, branch)
if setbranch:
self.after()
try:
self.repo = hg.repository(self.ui, branchpath)
except Exception:
self.repo = hg.repository(self.ui, branchpath, create=True)
self.before()
# pbranches may bring revisions from other branches (merge parents)
# Make sure we have them, or pull them.
missings = {}
for b in pbranches:
try:
self.repo.lookup(b[0])
except Exception:
missings.setdefault(b[1], []).append(b[0])
if missings:
self.after()
for pbranch, heads in sorted(missings.iteritems()):
pbranchpath = os.path.join(self.path, pbranch)
prepo = hg.peer(self.ui, {}, pbranchpath)
self.ui.note(
_('pulling from %s into %s\n') % (pbranch, branch))
exchange.pull(self.repo, prepo,
[prepo.lookup(h) for h in heads])
self.before()
def _mirrorrepo(ui, repo, url):
"""Mirror a source repository into the .hg directory of another."""
u = util.url(url)
if u.islocal():
raise error.Abort(_('source repo cannot be local'))
# Remove scheme from path and normalize reserved characters.
path = url.replace('%s://' % u.scheme, '').replace('/', '_')
mirrorpath = repo.vfs.join(store.encodefilename(path))
peer = hg.peer(ui, {}, url)
mirrorrepo = hg.repository(ui, mirrorpath,
create=not os.path.exists(mirrorpath))
missingheads = [head for head in peer.heads() if head not in mirrorrepo]
if missingheads:
ui.write(_('pulling %s into %s\n' % (url, mirrorpath)))
exchange.pull(mirrorrepo, peer)
return mirrorrepo
def setbranch(self, branch, pbranches):
if not self.clonebranches:
return
setbranch = (branch != self.lastbranch)
self.lastbranch = branch
if not branch:
branch = 'default'
pbranches = [(b[0], b[1] and b[1] or 'default') for b in pbranches]
if pbranches:
pbranch = pbranches[0][1]
else:
pbranch = 'default'
branchpath = os.path.join(self.path, branch)
if setbranch:
self.after()
try:
self.repo = hg.repository(self.ui, branchpath)
except Exception:
self.repo = hg.repository(self.ui, branchpath, create=True)
self.before()
# pbranches may bring revisions from other branches (merge parents)
# Make sure we have them, or pull them.
missings = {}
for b in pbranches:
try:
self.repo.lookup(b[0])
except Exception:
missings.setdefault(b[1], []).append(b[0])
if missings:
self.after()
for pbranch, heads in sorted(missings.iteritems()):
pbranchpath = os.path.join(self.path, pbranch)
prepo = hg.peer(self.ui, {}, pbranchpath)
self.ui.note(_('pulling from %s into %s\n') % (pbranch, branch))
exchange.pull(self.repo, prepo,
[prepo.lookup(h) for h in heads])
self.before()
def clone(self, remote, heads=[], stream=False):
supported = True
if not remote.capable('bundles'):
supported = False
self.ui.debug(_('bundle clone not supported\n'))
elif heads:
supported = False
self.ui.debug(_('cannot perform bundle clone if heads requested\n'))
if not supported:
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
result = remote._call('bundles')
if not result:
self.ui.note(_('no bundles available; using normal clone\n'))
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
# Eventually we'll support choosing the best options. Until then,
# use the first entry.
entry = result.splitlines()[0]
fields = entry.split()
url = fields[0]
if not url:
self.ui.note(_('invalid bundle manifest; using normal clone\n'))
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
self.ui.status(_('downloading bundle %s\n' % url))
try:
fh = hgurl.open(self.ui, url)
cg = exchange.readbundle(self.ui, fh, 'stream')
changegroup.addchangegroup(self, cg, 'bundleclone', url)
self.ui.status(_('finishing applying bundle; pulling\n'))
return exchange.pull(self, remote, heads=heads)
except urllib2.HTTPError as e:
self.ui.warn(_('HTTP error fetching bundle; using normal clone: %s\n') % str(e))
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
# This typically means a connectivity, DNS, etc problem.
except urllib2.URLError as e:
self.ui.warn(_('error fetching bundle; using normal clone: %s\n') % e.reason)
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
def _pullbundle(repo, rev):
"""Find the given rev in a backup bundle and pull it back into the
repository.
"""
other, rev = _findbundle(repo, rev)
if not other:
raise error.Abort("could not find '%s' in the repo or the backup"
" bundles" % rev)
lock = repo.lock()
try:
oldtip = len(repo)
exchange.pull(repo, other, heads=[rev])
tr = repo.transaction("phase")
nodes = (c.node() for c in repo.set('%d:', oldtip))
phases.retractboundary(repo, tr, 1, nodes)
tr.close()
finally:
lock.release()
if rev not in repo:
raise error.Abort("unable to get rev %s from repo" % rev)
return repo[rev]
def hg_pull(repo, peer, heads=None, force=False):
return exchange.pull(repo, peer, heads=heads, force=force)
def get_repo(url, alias):
global peer
myui = ui.ui()
myui.setconfig('ui', 'interactive', 'off')
myui.fout = sys.stderr
if get_config_bool('remote-hg.insecure'):
myui.setconfig('web', 'cacerts', '')
extensions.loadall(myui)
if hg.islocal(url) and not os.environ.get('GIT_REMOTE_HG_TEST_REMOTE'):
repo = hg.repository(myui, url)
if not os.path.exists(dirname):
os.makedirs(dirname)
else:
shared_path = os.path.join(gitdir, 'hg')
# check and upgrade old organization
hg_path = os.path.join(shared_path, '.hg')
if os.path.exists(shared_path) and not os.path.exists(hg_path):
repos = os.listdir(shared_path)
for x in repos:
local_hg = os.path.join(shared_path, x, 'clone', '.hg')
if not os.path.exists(local_hg):
continue
if not os.path.exists(hg_path):
shutil.move(local_hg, hg_path)
shutil.rmtree(os.path.join(shared_path, x, 'clone'))
# setup shared repo (if not there)
try:
hg.peer(myui, {}, shared_path, create=True)
except error.RepoError:
pass
if not os.path.exists(dirname):
os.makedirs(dirname)
local_path = os.path.join(dirname, 'clone')
if not os.path.exists(local_path):
hg.share(myui, shared_path, local_path, update=False)
else:
# make sure the shared path is always up-to-date
util.writefile(os.path.join(local_path, '.hg', 'sharedpath'), hg_path)
repo = hg.repository(myui, local_path)
try:
peer = hg.peer(repo.ui, {}, url)
except:
die('Repository error')
if check_version(3, 0):
from mercurial import exchange
exchange.pull(repo, peer, heads=None, force=True)
else:
repo.pull(peer, heads=None, force=True)
updatebookmarks(repo, peer)
return repo
def fetch(ui, repo, source=b'default', **opts):
"""pull changes from a remote repository, merge new changes if needed.
This finds all changes from the repository at the specified path
or URL and adds them to the local repository.
If the pulled changes add a new branch head, the head is
automatically merged, and the result of the merge is committed.
Otherwise, the working directory is updated to include the new
changes.
When a merge is needed, the working directory is first updated to
the newly pulled changes. Local changes are then merged into the
pulled changes. To switch the merge order, use --switch-parent.
See :hg:`help dates` for a list of formats valid for -d/--date.
Returns 0 on success.
"""
opts = pycompat.byteskwargs(opts)
date = opts.get(b'date')
if date:
opts[b'date'] = dateutil.parsedate(date)
parent = repo.dirstate.p1()
branch = repo.dirstate.branch()
try:
branchnode = repo.branchtip(branch)
except error.RepoLookupError:
branchnode = None
if parent != branchnode:
raise error.Abort(
_(b'working directory not at branch tip'),
hint=_(b"use 'hg update' to check out branch tip"),
)
wlock = lock = None
try:
wlock = repo.wlock()
lock = repo.lock()
cmdutil.bailifchanged(repo)
bheads = repo.branchheads(branch)
bheads = [head for head in bheads if len(repo[head].children()) == 0]
if len(bheads) > 1:
raise error.Abort(
_(b'multiple heads in this branch '
b'(use "hg heads ." and "hg merge" to merge)'))
other = hg.peer(repo, opts, ui.expandpath(source))
ui.status(
_(b'pulling from %s\n') % util.hidepassword(ui.expandpath(source)))
revs = None
if opts[b'rev']:
try:
revs = [other.lookup(rev) for rev in opts[b'rev']]
except error.CapabilityError:
err = _(b"other repository doesn't support revision lookup, "
b"so a rev cannot be specified.")
raise error.Abort(err)
# Are there any changes at all?
modheads = exchange.pull(repo, other, heads=revs).cgresult
if modheads == 0:
return 0
# Is this a simple fast-forward along the current branch?
newheads = repo.branchheads(branch)
newchildren = repo.changelog.nodesbetween([parent], newheads)[2]
if len(newheads) == 1 and len(newchildren):
if newchildren[0] != parent:
return hg.update(repo, newchildren[0])
else:
return 0
# Are there more than one additional branch heads?
newchildren = [n for n in newchildren if n != parent]
newparent = parent
if newchildren:
newparent = newchildren[0]
hg.clean(repo, newparent)
newheads = [n for n in newheads if n != newparent]
if len(newheads) > 1:
ui.status(
_(b'not merging with %d other new branch heads '
b'(use "hg heads ." and "hg merge" to merge them)\n') %
(len(newheads) - 1))
return 1
if not newheads:
return 0
# Otherwise, let's merge.
err = False
if newheads:
# By default, we consider the repository we're pulling
# *from* as authoritative, so we merge our changes into
# theirs.
if opts[b'switch_parent']:
firstparent, secondparent = newparent, newheads[0]
else:
firstparent, secondparent = newheads[0], newparent
ui.status(
_(b'updating to %d:%s\n') %
(repo.changelog.rev(firstparent), short(firstparent)))
hg.clean(repo, firstparent)
p2ctx = repo[secondparent]
ui.status(
_(b'merging with %d:%s\n') %
(p2ctx.rev(), short(secondparent)))
err = hg.merge(p2ctx, remind=False)
if not err:
# we don't translate commit messages
message = cmdutil.logmessage(
ui, opts) or (b'Automated merge with %s' %
util.removeauth(other.url()))
editopt = opts.get(b'edit') or opts.get(b'force_editor')
editor = cmdutil.getcommiteditor(edit=editopt, editform=b'fetch')
n = repo.commit(message,
opts[b'user'],
opts[b'date'],
editor=editor)
ui.status(
_(b'new changeset %d:%s merges remote changes with local\n') %
(repo.changelog.rev(n), short(n)))
return err
finally:
release(lock, wlock)
def clone(self, remote, heads=[], stream=False):
supported = True
if (exchange and hasattr(exchange, '_maybeapplyclonebundle')
and remote.capable('clonebundles')):
supported = False
self.ui.warn(
_('(mercurial client has built-in support for '
'bundle clone features; the "bundleclone" '
'extension can likely safely be removed)\n'))
if not self.ui.configbool('experimental', 'clonebundles',
False):
self.ui.warn(
_('(but the experimental.clonebundles config '
'flag is not enabled: enable it before '
'disabling bundleclone or cloning from '
'pre-generated bundles may not work)\n'))
# We assume that presence of the bundleclone extension
# means they want clonebundles enabled. Otherwise, why do
# they have bundleclone enabled? So silently enable it.
ui.setconfig('experimental', 'clonebundles', True)
elif not remote.capable('bundles'):
supported = False
self.ui.debug(_('bundle clone not supported\n'))
elif heads:
supported = False
self.ui.debug(
_('cannot perform bundle clone if heads requested\n'))
elif stream:
supported = False
self.ui.debug(
_('ignoring bundle clone because stream was '
'requested\n'))
if not supported:
return super(bundleclonerepo, self).clone(remote,
heads=heads,
stream=stream)
result = remote._call('bundles')
if not result:
self.ui.note(_('no bundles available; using normal clone\n'))
return super(bundleclonerepo, self).clone(remote,
heads=heads,
stream=stream)
pyver = sys.version_info
pyver = (pyver[0], pyver[1], pyver[2])
hgver = util.version()
# Discard bit after '+'.
hgver = hgver.split('+')[0]
try:
hgver = tuple([int(i) for i in hgver.split('.')[0:2]])
except ValueError:
hgver = (0, 0)
# Testing backdoors.
if ui.config('bundleclone', 'fakepyver'):
pyver = ui.configlist('bundleclone', 'fakepyver')
pyver = tuple(int(v) for v in pyver)
if ui.config('bundleclone', 'fakehgver'):
hgver = ui.configlist('bundleclone', 'fakehgver')
hgver = tuple(int(v) for v in hgver[0:2])
entries = []
snifilteredfrompython = False
snifilteredfromhg = False
for line in result.splitlines():
fields = line.split()
url = fields[0]
attrs = {}
for rawattr in fields[1:]:
key, value = rawattr.split('=', 1)
attrs[urllib.unquote(key)] = urllib.unquote(value)
# Filter out SNI entries if we don't support SNI.
if attrs.get('requiresni') == 'true':
skip = False
if pyver < (2, 7, 9):
# Take this opportunity to inform people they are using an
# old, insecure Python.
if not snifilteredfrompython:
self.ui.warn(
_('(your Python is older than 2.7.9 '
'and does not support modern and '
'secure SSL/TLS; please consider '
'upgrading your Python to a secure '
'version)\n'))
snifilteredfrompython = True
skip = True
if hgver < (3, 3):
if not snifilteredfromhg:
self.ui.warn(
_('(you Mercurial is old and does '
'not support modern and secure '
'SSL/TLS; please consider '
'upgrading your Mercurial to 3.3+ '
'which supports modern and secure '
'SSL/TLS)\n'))
snifilteredfromhg = True
skip = True
if skip:
self.ui.warn(
_('(ignoring URL on server that requires '
'SNI)\n'))
continue
entries.append((url, attrs))
if not entries:
# Don't fall back to normal clone because we don't want mass
# fallback in the wild to barage servers expecting bundle
# offload.
raise util.Abort(_('no appropriate bundles available'),
hint=_('you may wish to complain to the '
'server operator'))
# The configuration is allowed to define lists of preferred
# attributes and values. If this is present, sort results according
# to that preference. Otherwise, use manifest order and select the
# first entry.
prefers = self.ui.configlist('bundleclone', 'prefers', default=[])
if prefers:
prefers = [p.split('=', 1) for p in prefers]
def compareentry(a, b):
aattrs = a[1]
battrs = b[1]
# Itereate over local preferences.
for pkey, pvalue in prefers:
avalue = aattrs.get(pkey)
bvalue = battrs.get(pkey)
# Special case for b is missing attribute and a matches
# exactly.
if avalue is not None and bvalue is None and avalue == pvalue:
return -1
# Special case for a missing attribute and b matches
# exactly.
if bvalue is not None and avalue is None and bvalue == pvalue:
return 1
# We can't compare unless the attribute is defined on
# both entries.
if avalue is None or bvalue is None:
continue
# Same values should fall back to next attribute.
if avalue == bvalue:
continue
# Exact matches come first.
if avalue == pvalue:
return -1
if bvalue == pvalue:
return 1
# Fall back to next attribute.
continue
# Entries could not be sorted based on attributes. This
# says they are equal, which will fall back to index order,
# which is what we want.
return 0
entries = sorted(entries, cmp=compareentry)
url, attrs = entries[0]
if not url:
self.ui.note(
_('invalid bundle manifest; using normal clone\n'))
return super(bundleclonerepo, self).clone(remote,
heads=heads,
stream=stream)
self.ui.status(_('downloading bundle %s\n' % url))
try:
fh = hgurl.open(self.ui, url)
# Stream clone data is not changegroup data. Handle it
# specially.
if 'stream' in attrs:
reqs = set(attrs['stream'].split(','))
l = fh.readline()
filecount, bytecount = map(int, l.split(' ', 1))
self.ui.status(_('streaming all changes\n'))
consumev1(self, fh, filecount, bytecount)
else:
if exchange:
cg = exchange.readbundle(self.ui, fh, 'stream')
else:
cg = changegroup.readbundle(fh, 'stream')
# Mercurial 3.6 introduced cgNunpacker.apply().
# Before that, there was changegroup.addchangegroup().
# Before that, there was localrepository.addchangegroup().
if hasattr(cg, 'apply'):
cg.apply(self, 'bundleclone', url)
elif hasattr(changegroup, 'addchangegroup'):
changegroup.addchangegroup(self, cg, 'bundleclone',
url)
else:
self.addchangegroup(cg, 'bundleclone', url)
self.ui.status(_('finishing applying bundle; pulling\n'))
# Maintain compatibility with Mercurial 2.x.
if exchange:
return exchange.pull(self, remote, heads=heads)
else:
return self.pull(remote, heads=heads)
except (urllib2.HTTPError, urllib2.URLError) as e:
if isinstance(e, urllib2.HTTPError):
msg = _('HTTP error fetching bundle: %s') % str(e)
else:
msg = _('error fetching bundle: %s') % e.reason
# Don't fall back to regular clone unless explicitly told to.
if not self.ui.configbool('bundleclone', 'fallbackonerror',
False):
raise util.Abort(
msg,
hint=_('consider contacting the '
'server operator if this error persists'))
self.ui.warn(msg + '\n')
self.ui.warn(_('falling back to normal clone\n'))
return super(bundleclonerepo, self).clone(remote,
heads=heads,
stream=stream)
def _docheckout(ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
networkattemptlimit,
networkattempts=None,
sparse_profile=None):
if not networkattempts:
networkattempts = [1]
def callself():
return _docheckout(ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
networkattemptlimit,
networkattempts=networkattempts,
sparse_profile=sparse_profile)
ui.write('ensuring %s@%s is available at %s\n' %
(url, revision or branch, dest))
# We assume that we're the only process on the machine touching the
# repository paths that we were told to use. This means our recovery
# scenario when things aren't "right" is to just nuke things and start
# from scratch. This is easier to implement than verifying the state
# of the data and attempting recovery. And in some scenarios (such as
# potential repo corruption), it is probably faster, since verifying
# repos can take a while.
destvfs = getvfs()(dest, audit=False, realpath=True)
def deletesharedstore(path=None):
storepath = path or destvfs.read('.hg/sharedpath').strip()
if storepath.endswith('.hg'):
storepath = os.path.dirname(storepath)
storevfs = getvfs()(storepath, audit=False)
storevfs.rmtree(forcibly=True)
if destvfs.exists() and not destvfs.exists('.hg'):
raise error.Abort('destination exists but no .hg directory')
# Refuse to enable sparse checkouts on existing checkouts. The reasoning
# here is that another consumer of this repo may not be sparse aware. If we
# enabled sparse, we would lock them out.
if destvfs.exists(
) and sparse_profile and not destvfs.exists('.hg/sparse'):
raise error.Abort(
'cannot enable sparse profile on existing '
'non-sparse checkout',
hint='use a separate working directory to use sparse')
# And the other direction for symmetry.
if not sparse_profile and destvfs.exists('.hg/sparse'):
raise error.Abort(
'cannot use non-sparse checkout on existing sparse '
'checkout',
hint='use a separate working directory to use sparse')
# Require checkouts to be tied to shared storage because efficiency.
if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
ui.warn('(destination is not shared; deleting)\n')
destvfs.rmtree(forcibly=True)
# Verify the shared path exists and is using modern pooled storage.
if destvfs.exists('.hg/sharedpath'):
storepath = destvfs.read('.hg/sharedpath').strip()
ui.write('(existing repository shared store: %s)\n' % storepath)
if not os.path.exists(storepath):
ui.warn('(shared store does not exist; deleting destination)\n')
destvfs.rmtree(forcibly=True)
elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
ui.warn('(shared store does not belong to pooled storage; '
'deleting destination to improve efficiency)\n')
destvfs.rmtree(forcibly=True)
if destvfs.isfileorlink('.hg/wlock'):
ui.warn('(dest has an active working directory lock; assuming it is '
'left over from a previous process and that the destination '
'is corrupt; deleting it just to be sure)\n')
destvfs.rmtree(forcibly=True)
def handlerepoerror(e):
if e.message == _('abandoned transaction found'):
ui.warn('(abandoned transaction found; trying to recover)\n')
repo = hg.repository(ui, dest)
if not repo.recover():
ui.warn('(could not recover repo state; '
'deleting shared store)\n')
deletesharedstore()
ui.warn('(attempting checkout from beginning)\n')
return callself()
raise
# At this point we either have an existing working directory using
# shared, pooled storage or we have nothing.
def handlenetworkfailure():
if networkattempts[0] >= networkattemptlimit:
raise error.Abort('reached maximum number of network attempts; '
'giving up\n')
ui.warn('(retrying after network failure on attempt %d of %d)\n' %
(networkattempts[0], networkattemptlimit))
# Do a backoff on retries to mitigate the thundering herd
# problem. This is an exponential backoff with a multipler
# plus random jitter thrown in for good measure.
# With the default settings, backoffs will be:
# 1) 2.5 - 6.5
# 2) 5.5 - 9.5
# 3) 11.5 - 15.5
backoff = (2**networkattempts[0] - 1) * 1.5
jittermin = ui.configint('robustcheckout', 'retryjittermin', 1000)
jittermax = ui.configint('robustcheckout', 'retryjittermax', 5000)
backoff += float(random.randint(jittermin, jittermax)) / 1000.0
ui.warn('(waiting %.2fs before retry)\n' % backoff)
time.sleep(backoff)
networkattempts[0] += 1
def handlepullerror(e):
"""Handle an exception raised during a pull.
Returns True if caller should call ``callself()`` to retry.
"""
if isinstance(e, error.Abort):
if e.args[0] == _('repository is unrelated'):
ui.warn('(repository is unrelated; deleting)\n')
destvfs.rmtree(forcibly=True)
return True
elif e.args[0].startswith(_('stream ended unexpectedly')):
ui.warn('%s\n' % e.args[0])
# Will raise if failure limit reached.
handlenetworkfailure()
return True
elif isinstance(e, ssl.SSLError):
# Assume all SSL errors are due to the network, as Mercurial
# should convert non-transport errors like cert validation failures
# to error.Abort.
ui.warn('ssl error: %s\n' % e)
handlenetworkfailure()
return True
elif isinstance(e, urllib2.URLError):
if isinstance(e.reason, socket.error):
ui.warn('socket error: %s\n' % e.reason)
handlenetworkfailure()
return True
else:
ui.warn('unhandled URLError; reason type: %s; value: %s' %
(e.reason.__class__.__name__, e.reason))
else:
ui.warn('unhandled exception during network operation; type: %s; '
'value: %s' % (e.__class__.__name__, e))
return False
# Perform sanity checking of store. We may or may not know the path to the
# local store. It depends if we have an existing destvfs pointing to a
# share. To ensure we always find a local store, perform the same logic
# that Mercurial's pooled storage does to resolve the local store path.
cloneurl = upstream or url
try:
clonepeer = hg.peer(ui, {}, cloneurl)
rootnode = clonepeer.lookup('0')
except error.RepoLookupError:
raise error.Abort('unable to resolve root revision from clone '
'source')
except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
if rootnode == nullid:
raise error.Abort('source repo appears to be empty')
storepath = os.path.join(sharebase, hex(rootnode))
storevfs = getvfs()(storepath, audit=False)
if storevfs.isfileorlink('.hg/store/lock'):
ui.warn('(shared store has an active lock; assuming it is left '
'over from a previous process and that the store is '
'corrupt; deleting store and destination just to be '
'sure)\n')
if destvfs.exists():
destvfs.rmtree(forcibly=True)
storevfs.rmtree(forcibly=True)
if storevfs.exists() and not storevfs.exists('.hg/requires'):
ui.warn('(shared store missing requires file; this is a really '
'odd failure; deleting store and destination)\n')
if destvfs.exists():
destvfs.rmtree(forcibly=True)
storevfs.rmtree(forcibly=True)
if storevfs.exists('.hg/requires'):
requires = set(storevfs.read('.hg/requires').splitlines())
# FUTURE when we require generaldelta, this is where we can check
# for that.
required = {'dotencode', 'fncache'}
missing = required - requires
if missing:
ui.warn('(shared store missing requirements: %s; deleting '
'store and destination to ensure optimal behavior)\n' %
', '.join(sorted(missing)))
if destvfs.exists():
destvfs.rmtree(forcibly=True)
storevfs.rmtree(forcibly=True)
created = False
if not destvfs.exists():
# Ensure parent directories of destination exist.
# Mercurial 3.8 removed ensuredirs and made makedirs race safe.
if util.safehasattr(util, 'ensuredirs'):
makedirs = util.ensuredirs
else:
makedirs = util.makedirs
makedirs(os.path.dirname(destvfs.base), notindexed=True)
makedirs(sharebase, notindexed=True)
if upstream:
ui.write('(cloning from upstream repo %s)\n' % upstream)
try:
res = hg.clone(ui, {},
clonepeer,
dest=dest,
update=False,
shareopts={
'pool': sharebase,
'mode': 'identity'
})
except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' %
e.message)
deletesharedstore()
return callself()
# TODO retry here.
if res is None:
raise error.Abort('clone failed')
# Verify it is using shared pool storage.
if not destvfs.exists('.hg/sharedpath'):
raise error.Abort('clone did not create a shared repo')
created = True
# The destination .hg directory should exist. Now make sure we have the
# wanted revision.
repo = hg.repository(ui, dest)
# We only pull if we are using symbolic names or the requested revision
# doesn't exist.
havewantedrev = False
if revision and revision in repo:
ctx = repo[revision]
if not ctx.hex().startswith(revision):
raise error.Abort('--revision argument is ambiguous',
hint='must be the first 12+ characters of a '
'SHA-1 fragment')
checkoutrevision = ctx.hex()
havewantedrev = True
if not havewantedrev:
ui.write('(pulling to obtain %s)\n' % (revision or branch, ))
remote = None
try:
remote = hg.peer(repo, {}, url)
pullrevs = [remote.lookup(revision or branch)]
checkoutrevision = hex(pullrevs[0])
if branch:
ui.warn('(remote resolved %s to %s; '
'result is not deterministic)\n' %
(branch, checkoutrevision))
if checkoutrevision in repo:
ui.warn('(revision already present locally; not pulling)\n')
else:
pullop = exchange.pull(repo, remote, heads=pullrevs)
if not pullop.rheads:
raise error.Abort('unable to pull requested revision')
except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' %
e.message)
deletesharedstore()
return callself()
finally:
if remote:
remote.close()
# Now we should have the wanted revision in the store. Perform
# working directory manipulation.
# Purge if requested. We purge before update because this way we're
# guaranteed to not have conflicts on `hg update`.
if purge and not created:
ui.write('(purging working directory)\n')
purgeext = extensions.find('purge')
# Mercurial 4.3 doesn't purge files outside the sparse checkout.
# See https://bz.mercurial-scm.org/show_bug.cgi?id=5626. Force
# purging by monkeypatching the sparse matcher.
try:
old_sparse_fn = getattr(repo.dirstate, '_sparsematchfn', None)
if old_sparse_fn is not None:
assert util.versiontuple(n=2) in ((4, 3), (4, 4), (4, 5))
repo.dirstate._sparsematchfn = lambda: matchmod.always(
repo.root, '')
if purgeext.purge(
ui,
repo,
all=True,
abort_on_err=True,
# The function expects all arguments to be
# defined.
**{
'print': None,
'print0': None,
'dirs': None,
'files': None
}):
raise error.Abort('error purging')
finally:
if old_sparse_fn is not None:
repo.dirstate._sparsematchfn = old_sparse_fn
# Update the working directory.
if sparse_profile:
sparsemod = getsparse()
# By default, Mercurial will ignore unknown sparse profiles. This could
# lead to a full checkout. Be more strict.
try:
repo.filectx(sparse_profile, changeid=checkoutrevision).data()
except error.ManifestLookupError:
raise error.Abort('sparse profile %s does not exist at revision '
'%s' % (sparse_profile, checkoutrevision))
old_config = sparsemod.parseconfig(repo.ui, repo.vfs.tryread('sparse'))
old_includes, old_excludes, old_profiles = old_config
if old_profiles == {sparse_profile} and not old_includes and not \
old_excludes:
ui.write('(sparse profile %s already set; no need to update '
'sparse config)\n' % sparse_profile)
else:
if old_includes or old_excludes or old_profiles:
ui.write('(replacing existing sparse config with profile '
'%s)\n' % sparse_profile)
else:
ui.write('(setting sparse config to profile %s)\n' %
sparse_profile)
# If doing an incremental update, this will perform two updates:
# one to change the sparse profile and another to update to the new
# revision. This is not desired. But there's not a good API in
# Mercurial to do this as one operation.
with repo.wlock():
fcounts = map(
len,
sparsemod._updateconfigandrefreshwdir(repo, [], [],
[sparse_profile],
force=True))
repo.ui.status('%d files added, %d files dropped, '
'%d files conflicting\n' % tuple(fcounts))
ui.write('(sparse refresh complete)\n')
if commands.update(ui, repo, rev=checkoutrevision, clean=True):
raise error.Abort('error updating')
ui.write('updated to %s\n' % checkoutrevision)
return None
def _widen(
ui,
repo,
remote,
commoninc,
oldincludes,
oldexcludes,
newincludes,
newexcludes,
):
# for now we assume that if a server has ellipses enabled, we will be
# exchanging ellipses nodes. In future we should add ellipses as a client
# side requirement (maybe) to distinguish a client is shallow or not and
# then send that information to server whether we want ellipses or not.
# Theoretically a non-ellipses repo should be able to use narrow
# functionality from an ellipses enabled server
remotecap = remote.capabilities()
ellipsesremote = any(cap in remotecap
for cap in wireprototypes.SUPPORTED_ELLIPSESCAP)
# check whether we are talking to a server which supports old version of
# ellipses capabilities
isoldellipses = (ellipsesremote
and wireprototypes.ELLIPSESCAP1 in remotecap
and wireprototypes.ELLIPSESCAP not in remotecap)
def pullbundle2extraprepare_widen(orig, pullop, kwargs):
orig(pullop, kwargs)
# The old{in,ex}cludepats have already been set by orig()
kwargs[b'includepats'] = newincludes
kwargs[b'excludepats'] = newexcludes
wrappedextraprepare = extensions.wrappedfunction(
exchange, b'_pullbundle2extraprepare', pullbundle2extraprepare_widen)
# define a function that narrowbundle2 can call after creating the
# backup bundle, but before applying the bundle from the server
def setnewnarrowpats():
repo.setnarrowpats(newincludes, newexcludes)
repo.setnewnarrowpats = setnewnarrowpats
# silence the devel-warning of applying an empty changegroup
overrides = {(b'devel', b'all-warnings'): False}
common = commoninc[0]
with ui.uninterruptible():
if ellipsesremote:
ds = repo.dirstate
p1, p2 = ds.p1(), ds.p2()
with ds.parentchange():
ds.setparents(node.nullid, node.nullid)
if isoldellipses:
with wrappedextraprepare:
exchange.pull(repo, remote, heads=common)
else:
known = []
if ellipsesremote:
known = [
ctx.node() for ctx in repo.set(b'::%ln', common)
if ctx.node() != node.nullid
]
with remote.commandexecutor() as e:
bundle = e.callcommand(
b'narrow_widen',
{
b'oldincludes': oldincludes,
b'oldexcludes': oldexcludes,
b'newincludes': newincludes,
b'newexcludes': newexcludes,
b'cgversion': b'03',
b'commonheads': common,
b'known': known,
b'ellipses': ellipsesremote,
},
).result()
trmanager = exchange.transactionmanager(repo, b'widen',
remote.url())
with trmanager, repo.ui.configoverride(overrides, b'widen'):
op = bundle2.bundleoperation(repo,
trmanager.transaction,
source=b'widen')
# TODO: we should catch error.Abort here
bundle2.processbundle(repo, bundle, op=op)
if ellipsesremote:
with ds.parentchange():
ds.setparents(p1, p2)
with repo.transaction(b'widening'):
repo.setnewnarrowpats()
narrowspec.updateworkingcopy(repo)
narrowspec.copytoworkingcopy(repo)
def unifyrepo(ui, settings, **opts):
"""Unify the contents of multiple source repositories using settings.
The settings file is a Mercurial config file (basically an INI file).
"""
conf = unifyconfig(settings)
# Ensure destrepo is created with generaldelta enabled.
ui.setconfig('format', 'usegeneraldelta', True)
ui.setconfig('format', 'generaldelta', True)
# Verify all source repos have the same revision 0
rev0s = set()
for source in conf.sources:
repo = hg.repository(ui, path=source['path'])
# Verify
node = repo[0].node()
if rev0s and node not in rev0s:
raise error.Abort('repository has different rev 0: %s\n' %
source['name'])
# Verify pushlog exists
pushlog = getattr(repo, 'pushlog', None)
if not pushlog:
raise error.Abort('pushlog API not available',
hint='is the pushlog extension loaded?')
rev0s.add(node)
# Ensure the staging repo has all changesets from the source repos.
stageui = ui.copy()
# Now collect all the changeset data with pushlog info.
# node -> (when, source, rev, who, pushid)
nodepushinfo = {}
pushcount = 0
allnodes = set()
# Obtain pushlog data from each source repo. We obtain data for every node
# and filter later because we want to be sure we have the earliest known
# push data for a given node.
for source in conf.sources:
path = source['path']
sourcerepo = hg.repository(ui, path=source['path'])
pushlog = getattr(sourcerepo, 'pushlog', None)
index = sourcerepo.changelog.index
revnode = {}
for rev in sourcerepo:
# revlog.node() is too slow. Use the index directly.
node = index[rev][7]
revnode[rev] = node
allnodes.add(node)
noderev = {v: k for k, v in revnode.iteritems()}
localpushcount = 0
pushnodecount = 0
for pushid, who, when, nodes in pushlog.pushes():
pushcount += 1
localpushcount += 1
for node in nodes:
pushnodecount += 1
bnode = bin(node)
# There is a race between us iterating the repo and querying the
# pushlog. A new changeset could be written between when we
# obtain nodes and encounter the pushlog. So ignore pushlog
# for nodes we don't know about.
if bnode not in noderev:
ui.warn('pushlog entry for unknown node: %s; '
'possible race condition?\n' % node)
continue
rev = noderev[bnode]
if bnode not in nodepushinfo:
nodepushinfo[bnode] = (when, path, rev, who, pushid)
else:
currentwhen = nodepushinfo[bnode][0]
if when < currentwhen:
nodepushinfo[bnode] = (when, path, rev, who, pushid)
ui.write(
'obtained pushlog info for %d/%d revisions from %d pushes from %s\n'
% (pushnodecount, len(revnode), localpushcount, source['name']))
# Now verify that every node in the source repos has pushlog data.
missingpl = allnodes - set(nodepushinfo.keys())
if missingpl:
raise error.Abort(
'missing pushlog info for %d nodes: %s\n' %
(len(missingpl), ', '.join(sorted(hex(n) for n in missingpl))))
# Filter out changesets we aren't aggregating.
# We also use this pass to identify which nodes to bookmark.
books = {}
sourcenodes = set()
for source in conf.sources:
sourcerepo = hg.repository(ui, path=source['path'])
cl = sourcerepo.changelog
index = cl.index
sourcerevs = sourcerepo.revs(source['pullrevs'])
sourcerevs.sort()
headrevs = set(cl.headrevs())
sourceheadrevs = headrevs & set(sourcerevs)
# We /could/ allow multiple heads from each source repo. But for now
# it is easier to limit to 1 head per source.
if len(sourceheadrevs) > 1:
raise error.Abort(
'%s has %d heads' % (source['name'], len(sourceheadrevs)),
hint='define pullrevs to limit what is aggregated')
for rev in cl:
if rev not in sourcerevs:
continue
node = index[rev][7]
sourcenodes.add(node)
if source['bookmark']:
books[source['bookmark']] = node
ui.write(
'aggregating %d/%d revisions for %d heads from %s\n' %
(len(sourcerevs), len(cl), len(sourceheadrevs), source['name']))
nodepushinfo = {
k: v
for k, v in nodepushinfo.iteritems() if k in sourcenodes
}
ui.write('aggregating %d/%d nodes from %d original pushes\n' %
(len(nodepushinfo), len(allnodes), pushcount))
# We now have accounting for every changeset. Because pulling changesets
# is a bit time consuming, it is worthwhile to minimize the number of pull
# operations. We do this by ordering all changesets by original push time
# then emitting the minimum number of "fast forward" nodes from the tip
# of each linear range inside that list.
# (time, source, rev, user, pushid) -> node
inversenodeinfo = {v: k for k, v in nodepushinfo.iteritems()}
destui = ui.copy()
destui.setconfig('format', 'aggressivemergedeltas', True)
destui.setconfig('format', 'maxchainlen', 10000)
destrepo = hg.repository(destui,
path=conf.destpath,
create=not os.path.exists(conf.destpath))
destcl = destrepo.changelog
pullpushinfo = {
k: v
for k, v in inversenodeinfo.iteritems() if not destcl.hasnode(v)
}
ui.write('%d/%d nodes will be pulled\n' %
(len(pullpushinfo), len(inversenodeinfo)))
# Enable aggressive merge deltas on the stage repo to minimize manifest delta
# size. This could make delta chains very long. So we may want to institute a
# delta chain cap on the destination repo. But this will ensure the stage repo
# has the most efficient/compact representation of deltas. Pulling from this
# repo will also inherit the optimal delta, so we don't need to enable
# aggressivemergedeltas on the destination repo.
stageui.setconfig('format', 'aggressivemergedeltas', True)
stagerepo = hg.repository(stageui,
path=conf.stagepath,
create=not os.path.exists(conf.stagepath))
for source in conf.sources:
path = source['path']
sourcepeer = hg.peer(ui, {}, path)
ui.write('pulling %s into %s\n' % (path, conf.stagepath))
exchange.pull(stagerepo, sourcepeer)
pullnodes = list(emitfastforwardnodes(stagerepo, pullpushinfo))
unifiedpushes = list(unifypushes(inversenodeinfo))
ui.write('consolidated into %d pulls from %d unique pushes\n' %
(len(pullnodes), len(unifiedpushes)))
if not pullnodes:
ui.write('nothing to do; exiting\n')
return
stagepeer = hg.peer(ui, {}, conf.stagepath)
for node in pullnodes:
# TODO Bug 1265002 - we should update bookmarks when we pull.
# Otherwise the changesets will get replicated without a bookmark
# and any poor soul who pulls will see a nameless head.
exchange.pull(destrepo, stagepeer, heads=[node])
# For some reason there is a massive memory leak (10+ MB per
# iteration on Firefox repos) if we don't gc here.
gc.collect()
# Now that we've aggregated all the changesets in the destination repo,
# define the pushlog entries.
pushlog = getattr(destrepo, 'pushlog', None)
if not pushlog:
raise error.Abort('pushlog API not available',
hint='is the pushlog extension loaded?')
with destrepo.lock():
with destrepo.transaction('pushlog') as tr:
insertpushes = list(newpushes(destrepo, unifiedpushes))
ui.write('inserting %d pushlog entries\n' % len(insertpushes))
pushlog.recordpushes(insertpushes, tr=tr)
# Verify that pushlog time in revision order is always increasing.
destnodepushtime = {}
for push in destrepo.pushlog.pushes():
for node in push.nodes:
destnodepushtime[bin(node)] = push.when
destcl = destrepo.changelog
lastpushtime = 0
for rev in destrepo:
node = destcl.node(rev)
pushtime = destnodepushtime[node]
if pushtime < lastpushtime:
ui.warn('push time for %d is older than %d\n' % (rev, rev - 1))
lastpushtime = pushtime
# Write bookmarks.
ui.write('writing %d bookmarks\n' % len(books))
with destrepo.wlock():
with destrepo.lock():
with destrepo.transaction('bookmarks') as tr:
bm = bookmarks.bmstore(destrepo)
books.update({
book: None # delete any bookmarks not found in the update
for book in bm.keys() if book not in books
})
# Mass replacing may not be the proper strategy. But it works for
# our current use case.
bm.applychanges(destrepo, tr, books.items())
if not opts.get('skipreplicate'):
# This is a bit hacky. Pushlog and bookmarks aren't currently replicated
# via the normal hooks mechanism because we use the low-level APIs to
# write them. So, we send a replication message to sync the entire repo.
try:
vcsr = extensions.find('vcsreplicator')
except KeyError:
raise error.Abort(
'vcsreplicator extension not installed; '
'pushlog and bookmarks may not be replicated properly')
vcsr.replicatecommand(destrepo.ui, destrepo)
def _do_case(self, name, layout):
subdir = test_util.subdir.get(name, '')
single = layout == 'single'
u = test_util.testui()
config = {}
if layout == 'custom':
for branch, path in test_util.custom.get(name, {}).iteritems():
config['hgsubversionbranch.%s' % branch] = path
u.setconfig('hgsubversionbranch', branch, path)
repo, repo_path = self.load_and_fetch(name, subdir=subdir, layout=layout)
assert test_util.repolen(self.repo) > 0
wc2_path = self.wc_path + '_clone'
src, dest = test_util.hgclone(u, self.wc_path, wc2_path, update=False)
src = test_util.getlocalpeer(src)
dest = test_util.getlocalpeer(dest)
# insert a wrapper that prevents calling changectx.children()
def failfn(orig, ctx):
self.fail('calling %s is forbidden; it can cause massive slowdowns '
'when rebuilding large repositories' % orig)
origchildren = getattr(context.changectx, 'children')
extensions.wrapfunction(context.changectx, 'children', failfn)
try:
svncommands.rebuildmeta(u,
dest,
args=[
test_util.fileurl(repo_path + subdir),
])
finally:
# remove the wrapper
context.changectx.children = origchildren
self._run_assertions(name, single, src, dest, u)
wc3_path = self.wc_path + '_partial'
src, dest = test_util.hgclone(u,
self.wc_path,
wc3_path,
update=False,
rev=['0'])
srcrepo = test_util.getlocalpeer(src)
dest = test_util.getlocalpeer(dest)
# insert a wrapper that prevents calling changectx.children()
extensions.wrapfunction(context.changectx, 'children', failfn)
try:
svncommands.rebuildmeta(u,
dest,
args=[
test_util.fileurl(repo_path + subdir),
])
finally:
# remove the wrapper
context.changectx.children = origchildren
if hgutil.safehasattr(localrepo.localrepository, 'pull'):
dest.pull(src)
else:
# Mercurial >= 3.2
from mercurial import exchange
exchange.pull(dest, src)
# insert a wrapper that prevents calling changectx.children()
extensions.wrapfunction(context.changectx, 'children', failfn)
try:
svncommands.updatemeta(u,
dest,
args=[
test_util.fileurl(repo_path + subdir),
])
finally:
# remove the wrapper
context.changectx.children = origchildren
self._run_assertions(name, single, srcrepo, dest, u)
def _docheckout(ui, url, dest, upstream, revision, branch, purge, sharebase,
networkattemptlimit, networkattempts=None, sparse_profile=None):
if not networkattempts:
networkattempts = [1]
def callself():
return _docheckout(ui, url, dest, upstream, revision, branch, purge,
sharebase, networkattemptlimit,
networkattempts=networkattempts,
sparse_profile=sparse_profile)
ui.write('ensuring %s@%s is available at %s\n' % (url, revision or branch,
dest))
# We assume that we're the only process on the machine touching the
# repository paths that we were told to use. This means our recovery
# scenario when things aren't "right" is to just nuke things and start
# from scratch. This is easier to implement than verifying the state
# of the data and attempting recovery. And in some scenarios (such as
# potential repo corruption), it is probably faster, since verifying
# repos can take a while.
destvfs = getvfs()(dest, audit=False, realpath=True)
def deletesharedstore(path=None):
storepath = path or destvfs.read('.hg/sharedpath').strip()
if storepath.endswith('.hg'):
storepath = os.path.dirname(storepath)
storevfs = getvfs()(storepath, audit=False)
storevfs.rmtree(forcibly=True)
if destvfs.exists() and not destvfs.exists('.hg'):
raise error.Abort('destination exists but no .hg directory')
# Refuse to enable sparse checkouts on existing checkouts. The reasoning
# here is that another consumer of this repo may not be sparse aware. If we
# enabled sparse, we would lock them out.
if destvfs.exists() and sparse_profile and not destvfs.exists('.hg/sparse'):
raise error.Abort('cannot enable sparse profile on existing '
'non-sparse checkout',
hint='use a separate working directory to use sparse')
# And the other direction for symmetry.
if not sparse_profile and destvfs.exists('.hg/sparse'):
raise error.Abort('cannot use non-sparse checkout on existing sparse '
'checkout',
hint='use a separate working directory to use sparse')
# Require checkouts to be tied to shared storage because efficiency.
if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
ui.warn('(destination is not shared; deleting)\n')
destvfs.rmtree(forcibly=True)
# Verify the shared path exists and is using modern pooled storage.
if destvfs.exists('.hg/sharedpath'):
storepath = destvfs.read('.hg/sharedpath').strip()
ui.write('(existing repository shared store: %s)\n' % storepath)
if not os.path.exists(storepath):
ui.warn('(shared store does not exist; deleting destination)\n')
destvfs.rmtree(forcibly=True)
elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
ui.warn('(shared store does not belong to pooled storage; '
'deleting destination to improve efficiency)\n')
destvfs.rmtree(forcibly=True)
if destvfs.isfileorlink('.hg/wlock'):
ui.warn('(dest has an active working directory lock; assuming it is '
'left over from a previous process and that the destination '
'is corrupt; deleting it just to be sure)\n')
destvfs.rmtree(forcibly=True)
def handlerepoerror(e):
if e.message == _('abandoned transaction found'):
ui.warn('(abandoned transaction found; trying to recover)\n')
repo = hg.repository(ui, dest)
if not repo.recover():
ui.warn('(could not recover repo state; '
'deleting shared store)\n')
deletesharedstore()
ui.warn('(attempting checkout from beginning)\n')
return callself()
raise
# At this point we either have an existing working directory using
# shared, pooled storage or we have nothing.
def handlenetworkfailure():
if networkattempts[0] >= networkattemptlimit:
raise error.Abort('reached maximum number of network attempts; '
'giving up\n')
ui.warn('(retrying after network failure on attempt %d of %d)\n' %
(networkattempts[0], networkattemptlimit))
# Do a backoff on retries to mitigate the thundering herd
# problem. This is an exponential backoff with a multipler
# plus random jitter thrown in for good measure.
# With the default settings, backoffs will be:
# 1) 2.5 - 6.5
# 2) 5.5 - 9.5
# 3) 11.5 - 15.5
backoff = (2 ** networkattempts[0] - 1) * 1.5
jittermin = ui.configint('robustcheckout', 'retryjittermin', 1000)
jittermax = ui.configint('robustcheckout', 'retryjittermax', 5000)
backoff += float(random.randint(jittermin, jittermax)) / 1000.0
ui.warn('(waiting %.2fs before retry)\n' % backoff)
time.sleep(backoff)
networkattempts[0] += 1
def handlepullerror(e):
"""Handle an exception raised during a pull.
Returns True if caller should call ``callself()`` to retry.
"""
if isinstance(e, error.Abort):
if e.args[0] == _('repository is unrelated'):
ui.warn('(repository is unrelated; deleting)\n')
destvfs.rmtree(forcibly=True)
return True
elif e.args[0].startswith(_('stream ended unexpectedly')):
ui.warn('%s\n' % e.args[0])
# Will raise if failure limit reached.
handlenetworkfailure()
return True
elif isinstance(e, ssl.SSLError):
# Assume all SSL errors are due to the network, as Mercurial
# should convert non-transport errors like cert validation failures
# to error.Abort.
ui.warn('ssl error: %s\n' % e)
handlenetworkfailure()
return True
elif isinstance(e, urllib2.URLError):
if isinstance(e.reason, socket.error):
ui.warn('socket error: %s\n' % e.reason)
handlenetworkfailure()
return True
else:
ui.warn('unhandled URLError; reason type: %s; value: %s' % (
e.reason.__class__.__name__, e.reason))
else:
ui.warn('unhandled exception during network operation; type: %s; '
'value: %s' % (e.__class__.__name__, e))
return False
# Perform sanity checking of store. We may or may not know the path to the
# local store. It depends if we have an existing destvfs pointing to a
# share. To ensure we always find a local store, perform the same logic
# that Mercurial's pooled storage does to resolve the local store path.
cloneurl = upstream or url
try:
clonepeer = hg.peer(ui, {}, cloneurl)
rootnode = clonepeer.lookup('0')
except error.RepoLookupError:
raise error.Abort('unable to resolve root revision from clone '
'source')
except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
if rootnode == nullid:
raise error.Abort('source repo appears to be empty')
storepath = os.path.join(sharebase, hex(rootnode))
storevfs = getvfs()(storepath, audit=False)
if storevfs.isfileorlink('.hg/store/lock'):
ui.warn('(shared store has an active lock; assuming it is left '
'over from a previous process and that the store is '
'corrupt; deleting store and destination just to be '
'sure)\n')
if destvfs.exists():
destvfs.rmtree(forcibly=True)
storevfs.rmtree(forcibly=True)
if storevfs.exists() and not storevfs.exists('.hg/requires'):
ui.warn('(shared store missing requires file; this is a really '
'odd failure; deleting store and destination)\n')
if destvfs.exists():
destvfs.rmtree(forcibly=True)
storevfs.rmtree(forcibly=True)
if storevfs.exists('.hg/requires'):
requires = set(storevfs.read('.hg/requires').splitlines())
# FUTURE when we require generaldelta, this is where we can check
# for that.
required = {'dotencode', 'fncache'}
missing = required - requires
if missing:
ui.warn('(shared store missing requirements: %s; deleting '
'store and destination to ensure optimal behavior)\n' %
', '.join(sorted(missing)))
if destvfs.exists():
destvfs.rmtree(forcibly=True)
storevfs.rmtree(forcibly=True)
created = False
if not destvfs.exists():
# Ensure parent directories of destination exist.
# Mercurial 3.8 removed ensuredirs and made makedirs race safe.
if util.safehasattr(util, 'ensuredirs'):
makedirs = util.ensuredirs
else:
makedirs = util.makedirs
makedirs(os.path.dirname(destvfs.base), notindexed=True)
makedirs(sharebase, notindexed=True)
if upstream:
ui.write('(cloning from upstream repo %s)\n' % upstream)
try:
res = hg.clone(ui, {}, clonepeer, dest=dest, update=False,
shareopts={'pool': sharebase, 'mode': 'identity'})
except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
deletesharedstore()
return callself()
# TODO retry here.
if res is None:
raise error.Abort('clone failed')
# Verify it is using shared pool storage.
if not destvfs.exists('.hg/sharedpath'):
raise error.Abort('clone did not create a shared repo')
created = True
# The destination .hg directory should exist. Now make sure we have the
# wanted revision.
repo = hg.repository(ui, dest)
# We only pull if we are using symbolic names or the requested revision
# doesn't exist.
havewantedrev = False
if revision and revision in repo:
ctx = repo[revision]
if not ctx.hex().startswith(revision):
raise error.Abort('--revision argument is ambiguous',
hint='must be the first 12+ characters of a '
'SHA-1 fragment')
checkoutrevision = ctx.hex()
havewantedrev = True
if not havewantedrev:
ui.write('(pulling to obtain %s)\n' % (revision or branch,))
remote = None
try:
remote = hg.peer(repo, {}, url)
pullrevs = [remote.lookup(revision or branch)]
checkoutrevision = hex(pullrevs[0])
if branch:
ui.warn('(remote resolved %s to %s; '
'result is not deterministic)\n' %
(branch, checkoutrevision))
if checkoutrevision in repo:
ui.warn('(revision already present locally; not pulling)\n')
else:
pullop = exchange.pull(repo, remote, heads=pullrevs)
if not pullop.rheads:
raise error.Abort('unable to pull requested revision')
except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
deletesharedstore()
return callself()
finally:
if remote:
remote.close()
# Now we should have the wanted revision in the store. Perform
# working directory manipulation.
# Purge if requested. We purge before update because this way we're
# guaranteed to not have conflicts on `hg update`.
if purge and not created:
ui.write('(purging working directory)\n')
purgeext = extensions.find('purge')
# Mercurial 4.3 doesn't purge files outside the sparse checkout.
# See https://bz.mercurial-scm.org/show_bug.cgi?id=5626. Force
# purging by monkeypatching the sparse matcher.
try:
old_sparse_fn = getattr(repo.dirstate, '_sparsematchfn', None)
if old_sparse_fn is not None:
assert util.versiontuple(n=2) in ((4, 3), (4, 4), (4, 5))
repo.dirstate._sparsematchfn = lambda: matchmod.always(repo.root, '')
if purgeext.purge(ui, repo, all=True, abort_on_err=True,
# The function expects all arguments to be
# defined.
**{'print': None, 'print0': None, 'dirs': None,
'files': None}):
raise error.Abort('error purging')
finally:
if old_sparse_fn is not None:
repo.dirstate._sparsematchfn = old_sparse_fn
# Update the working directory.
if sparse_profile:
sparsemod = getsparse()
# By default, Mercurial will ignore unknown sparse profiles. This could
# lead to a full checkout. Be more strict.
try:
repo.filectx(sparse_profile, changeid=checkoutrevision).data()
except error.ManifestLookupError:
raise error.Abort('sparse profile %s does not exist at revision '
'%s' % (sparse_profile, checkoutrevision))
old_config = sparsemod.parseconfig(repo.ui, repo.vfs.tryread('sparse'))
old_includes, old_excludes, old_profiles = old_config
if old_profiles == {sparse_profile} and not old_includes and not \
old_excludes:
ui.write('(sparse profile %s already set; no need to update '
'sparse config)\n' % sparse_profile)
else:
if old_includes or old_excludes or old_profiles:
ui.write('(replacing existing sparse config with profile '
'%s)\n' % sparse_profile)
else:
ui.write('(setting sparse config to profile %s)\n' %
sparse_profile)
# If doing an incremental update, this will perform two updates:
# one to change the sparse profile and another to update to the new
# revision. This is not desired. But there's not a good API in
# Mercurial to do this as one operation.
with repo.wlock():
fcounts = map(len, sparsemod._updateconfigandrefreshwdir(
repo, [], [], [sparse_profile], force=True))
repo.ui.status('%d files added, %d files dropped, '
'%d files conflicting\n' % tuple(fcounts))
ui.write('(sparse refresh complete)\n')
if commands.update(ui, repo, rev=checkoutrevision, clean=True):
raise error.Abort('error updating')
ui.write('updated to %s\n' % checkoutrevision)
return None
def get_repo(url, alias):
global peer
myui = ui.ui()
myui.setconfig('ui', 'interactive', 'off')
myui.fout = sys.stderr
if get_config_bool('remote-hg.insecure'):
myui.setconfig('web', 'cacerts', '')
extensions.loadall(myui)
if hg.islocal(url) and not os.environ.get('GIT_REMOTE_HG_TEST_REMOTE'):
repo = hg.repository(myui, url)
if not os.path.exists(dirname):
os.makedirs(dirname)
else:
shared_path = os.path.join(gitdir, 'hg')
# check and upgrade old organization
hg_path = os.path.join(shared_path, '.hg')
if os.path.exists(shared_path) and not os.path.exists(hg_path):
repos = os.listdir(shared_path)
for x in repos:
local_hg = os.path.join(shared_path, x, 'clone', '.hg')
if not os.path.exists(local_hg):
continue
if not os.path.exists(hg_path):
shutil.move(local_hg, hg_path)
shutil.rmtree(os.path.join(shared_path, x, 'clone'))
# setup shared repo (if not there)
try:
hg.peer(myui, {}, shared_path, create=True)
except error.RepoError:
pass
if not os.path.exists(dirname):
os.makedirs(dirname)
local_path = os.path.join(dirname, 'clone')
if not os.path.exists(local_path):
hg.share(myui, shared_path, local_path, update=False)
else:
# make sure the shared path is always up-to-date
util.writefile(os.path.join(local_path, '.hg', 'sharedpath'), hg_path)
repo = hg.repository(myui, local_path)
try:
peer = hg.peer(repo.ui, {}, url)
except:
die('Repository error')
if check_version(3, 0):
from mercurial import exchange
exchange.pull(repo, peer, heads=None, force=True)
else:
repo.pull(peer, heads=None, force=True)
updatebookmarks(repo, peer)
return repo
def clone(self, remote, heads=[], stream=False):
supported = True
if not remote.capable('bundles'):
supported = False
self.ui.debug(_('bundle clone not supported\n'))
elif heads:
supported = False
self.ui.debug(_('cannot perform bundle clone if heads requested\n'))
elif stream:
supported = False
self.ui.debug(_('ignoring bundle clone because stream was '
'requested\n'))
if not supported:
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
result = remote._call('bundles')
if not result:
self.ui.note(_('no bundles available; using normal clone\n'))
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
pyver = sys.version_info
pyver = (pyver[0], pyver[1], pyver[2])
# Testing backdoor.
if ui.config('bundleclone', 'fakepyver'):
pyver = ui.configlist('bundleclone', 'fakepyver')
pyver = tuple(int(v) for v in pyver)
entries = []
snifiltered = False
for line in result.splitlines():
fields = line.split()
url = fields[0]
attrs = {}
for rawattr in fields[1:]:
key, value = rawattr.split('=', 1)
attrs[urllib.unquote(key)] = urllib.unquote(value)
# Filter out SNI entries if we don't support SNI.
if attrs.get('requiresni') == 'true' and pyver < (2, 7, 9):
# Take this opportunity to inform people they are using an
# old, insecure Python.
if not snifiltered:
self.ui.warn(_('(ignoring URL on server that requires '
'SNI)\n'))
self.ui.warn(_('(your Python is older than 2.7.9 and '
'does not support modern and secure '
'SSL/TLS; please consider upgrading '
'your Python to a secure version)\n'))
snifiltered = True
continue
entries.append((url, attrs))
if not entries:
# Don't fall back to normal clone because we don't want mass
# fallback in the wild to barage servers expecting bundle
# offload.
raise util.Abort(_('no appropriate bundles available'),
hint=_('you may wish to complain to the '
'server operator'))
# The configuration is allowed to define lists of preferred
# attributes and values. If this is present, sort results according
# to that preference. Otherwise, use manifest order and select the
# first entry.
prefers = self.ui.configlist('bundleclone', 'prefers', default=[])
if prefers:
prefers = [p.split('=', 1) for p in prefers]
def compareentry(a, b):
aattrs = a[1]
battrs = b[1]
# Itereate over local preferences.
for pkey, pvalue in prefers:
avalue = aattrs.get(pkey)
bvalue = battrs.get(pkey)
# Special case for b is missing attribute and a matches
# exactly.
if avalue is not None and bvalue is None and avalue == pvalue:
return -1
# Special case for a missing attribute and b matches
# exactly.
if bvalue is not None and avalue is None and bvalue == pvalue:
return 1
# We can't compare unless the attribute is defined on
# both entries.
if avalue is None or bvalue is None:
continue
# Same values should fall back to next attribute.
if avalue == bvalue:
continue
# Exact matches come first.
if avalue == pvalue:
return -1
if bvalue == pvalue:
return 1
# Fall back to next attribute.
continue
# Entries could not be sorted based on attributes. This
# says they are equal, which will fall back to index order,
# which is what we want.
return 0
entries = sorted(entries, cmp=compareentry)
url, attrs = entries[0]
if not url:
self.ui.note(_('invalid bundle manifest; using normal clone\n'))
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
self.ui.status(_('downloading bundle %s\n' % url))
try:
fh = hgurl.open(self.ui, url)
# Stream clone data is not changegroup data. Handle it
# specially.
if 'stream' in attrs:
reqs = set(attrs['stream'].split(','))
applystreamclone(self, reqs, fh)
else:
if exchange:
cg = exchange.readbundle(self.ui, fh, 'stream')
else:
cg = changegroup.readbundle(fh, 'stream')
if hasattr(changegroup, 'addchangegroup'):
changegroup.addchangegroup(self, cg, 'bundleclone', url)
else:
self.addchangegroup(cg, 'bundleclone', url)
self.ui.status(_('finishing applying bundle; pulling\n'))
# Maintain compatibility with Mercurial 2.x.
if exchange:
return exchange.pull(self, remote, heads=heads)
else:
return self.pull(remote, heads=heads)
except (urllib2.HTTPError, urllib2.URLError) as e:
if isinstance(e, urllib2.HTTPError):
msg = _('HTTP error fetching bundle: %s') % str(e)
else:
msg = _('error fetching bundle: %s') % e.reason
# Don't fall back to regular clone unless explicitly told to.
if not self.ui.configbool('bundleclone', 'fallbackonerror', False):
raise util.Abort(msg, hint=_('consider contacting the '
'server operator if this error persists'))
self.ui.warn(msg + '\n')
self.ui.warn(_('falling back to normal clone\n'))
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
def apply(self, repo, source, revmap, merges, opts={}):
'''apply the revisions in revmap one by one in revision order'''
revs = sorted(revmap)
p1, p2 = repo.dirstate.parents()
pulls = []
diffopts = patch.diffopts(self.ui, opts)
diffopts.git = True
lock = wlock = tr = None
try:
wlock = repo.wlock()
lock = repo.lock()
tr = repo.transaction('transplant')
for rev in revs:
node = revmap[rev]
revstr = '%s:%s' % (rev, short(node))
if self.applied(repo, node, p1):
self.ui.warn(_('skipping already applied revision %s\n') %
revstr)
continue
parents = source.changelog.parents(node)
if not (opts.get('filter') or opts.get('log')):
# If the changeset parent is the same as the
# wdir's parent, just pull it.
if parents[0] == p1:
pulls.append(node)
p1 = node
continue
if pulls:
if source != repo:
exchange.pull(repo, source.peer(), heads=pulls)
merge.update(repo, pulls[-1], False, False, None)
p1, p2 = repo.dirstate.parents()
pulls = []
domerge = False
if node in merges:
# pulling all the merge revs at once would mean we
# couldn't transplant after the latest even if
# transplants before them fail.
domerge = True
if not hasnode(repo, node):
exchange.pull(repo, source.peer(), heads=[node])
skipmerge = False
if parents[1] != revlog.nullid:
if not opts.get('parent'):
self.ui.note(_('skipping merge changeset %s:%s\n')
% (rev, short(node)))
skipmerge = True
else:
parent = source.lookup(opts['parent'])
if parent not in parents:
raise util.Abort(_('%s is not a parent of %s') %
(short(parent), short(node)))
else:
parent = parents[0]
if skipmerge:
patchfile = None
else:
fd, patchfile = tempfile.mkstemp(prefix='hg-transplant-')
fp = os.fdopen(fd, 'w')
gen = patch.diff(source, parent, node, opts=diffopts)
for chunk in gen:
fp.write(chunk)
fp.close()
del revmap[rev]
if patchfile or domerge:
try:
try:
n = self.applyone(repo, node,
source.changelog.read(node),
patchfile, merge=domerge,
log=opts.get('log'),
filter=opts.get('filter'))
except TransplantError:
# Do not rollback, it is up to the user to
# fix the merge or cancel everything
tr.close()
raise
if n and domerge:
self.ui.status(_('%s merged at %s\n') % (revstr,
short(n)))
elif n:
self.ui.status(_('%s transplanted to %s\n')
% (short(node),
short(n)))
finally:
if patchfile:
os.unlink(patchfile)
tr.close()
if pulls:
exchange.pull(repo, source.peer(), heads=pulls)
merge.update(repo, pulls[-1], False, False, None)
finally:
self.saveseries(revmap, merges)
self.transplants.write()
if tr:
tr.release()
lock.release()
wlock.release()
def dopull(self):
self.ui.status(_('pulling from %s\n') % util.hidepassword(self.ui.expandpath(self.source)))
exchange.pull(self.repo, self.remoterepository)
self.ui.write('pull complete\n')
def clone(self, remote, heads=[], stream=False):
supported = True
if (exchange and hasattr(exchange, '_maybeapplyclonebundle')
and remote.capable('clonebundles')):
supported = False
self.ui.warn(_('(mercurial client has built-in support for '
'bundle clone features; the "bundleclone" '
'extension can likely safely be removed)\n'))
if not self.ui.configbool('experimental', 'clonebundles', False):
self.ui.warn(_('(but the experimental.clonebundles config '
'flag is not enabled: enable it before '
'disabling bundleclone or cloning from '
'pre-generated bundles may not work)\n'))
# We assume that presence of the bundleclone extension
# means they want clonebundles enabled. Otherwise, why do
# they have bundleclone enabled? So silently enable it.
ui.setconfig('experimental', 'clonebundles', True)
elif not remote.capable('bundles'):
supported = False
self.ui.debug(_('bundle clone not supported\n'))
elif heads:
supported = False
self.ui.debug(_('cannot perform bundle clone if heads requested\n'))
elif stream:
supported = False
self.ui.debug(_('ignoring bundle clone because stream was '
'requested\n'))
if not supported:
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
result = remote._call('bundles')
if not result:
self.ui.note(_('no bundles available; using normal clone\n'))
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
pyver = sys.version_info
pyver = (pyver[0], pyver[1], pyver[2])
hgver = util.version()
# Discard bit after '+'.
hgver = hgver.split('+')[0]
try:
hgver = tuple([int(i) for i in hgver.split('.')[0:2]])
except ValueError:
hgver = (0, 0)
# Testing backdoors.
if ui.config('bundleclone', 'fakepyver'):
pyver = ui.configlist('bundleclone', 'fakepyver')
pyver = tuple(int(v) for v in pyver)
if ui.config('bundleclone', 'fakehgver'):
hgver = ui.configlist('bundleclone', 'fakehgver')
hgver = tuple(int(v) for v in hgver[0:2])
entries = []
snifilteredfrompython = False
snifilteredfromhg = False
for line in result.splitlines():
fields = line.split()
url = fields[0]
attrs = {}
for rawattr in fields[1:]:
key, value = rawattr.split('=', 1)
attrs[urllib.unquote(key)] = urllib.unquote(value)
# Filter out SNI entries if we don't support SNI.
if attrs.get('requiresni') == 'true':
skip = False
if pyver < (2, 7, 9):
# Take this opportunity to inform people they are using an
# old, insecure Python.
if not snifilteredfrompython:
self.ui.warn(_('(your Python is older than 2.7.9 '
'and does not support modern and '
'secure SSL/TLS; please consider '
'upgrading your Python to a secure '
'version)\n'))
snifilteredfrompython = True
skip = True
if hgver < (3, 3):
if not snifilteredfromhg:
self.ui.warn(_('(you Mercurial is old and does '
'not support modern and secure '
'SSL/TLS; please consider '
'upgrading your Mercurial to 3.3+ '
'which supports modern and secure '
'SSL/TLS)\n'))
snifilteredfromhg = True
skip = True
if skip:
self.ui.warn(_('(ignoring URL on server that requires '
'SNI)\n'))
continue
entries.append((url, attrs))
if not entries:
# Don't fall back to normal clone because we don't want mass
# fallback in the wild to barage servers expecting bundle
# offload.
raise util.Abort(_('no appropriate bundles available'),
hint=_('you may wish to complain to the '
'server operator'))
# The configuration is allowed to define lists of preferred
# attributes and values. If this is present, sort results according
# to that preference. Otherwise, use manifest order and select the
# first entry.
prefers = self.ui.configlist('bundleclone', 'prefers', default=[])
if prefers:
prefers = [p.split('=', 1) for p in prefers]
def compareentry(a, b):
aattrs = a[1]
battrs = b[1]
# Itereate over local preferences.
for pkey, pvalue in prefers:
avalue = aattrs.get(pkey)
bvalue = battrs.get(pkey)
# Special case for b is missing attribute and a matches
# exactly.
if avalue is not None and bvalue is None and avalue == pvalue:
return -1
# Special case for a missing attribute and b matches
# exactly.
if bvalue is not None and avalue is None and bvalue == pvalue:
return 1
# We can't compare unless the attribute is defined on
# both entries.
if avalue is None or bvalue is None:
continue
# Same values should fall back to next attribute.
if avalue == bvalue:
continue
# Exact matches come first.
if avalue == pvalue:
return -1
if bvalue == pvalue:
return 1
# Fall back to next attribute.
continue
# Entries could not be sorted based on attributes. This
# says they are equal, which will fall back to index order,
# which is what we want.
return 0
entries = sorted(entries, cmp=compareentry)
url, attrs = entries[0]
if not url:
self.ui.note(_('invalid bundle manifest; using normal clone\n'))
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
self.ui.status(_('downloading bundle %s\n' % url))
try:
fh = hgurl.open(self.ui, url)
# Stream clone data is not changegroup data. Handle it
# specially.
if 'stream' in attrs:
reqs = set(attrs['stream'].split(','))
l = fh.readline()
filecount, bytecount = map(int, l.split(' ', 1))
self.ui.status(_('streaming all changes\n'))
consumev1(self, fh, filecount, bytecount)
else:
if exchange:
cg = exchange.readbundle(self.ui, fh, 'stream')
else:
cg = changegroup.readbundle(fh, 'stream')
# Mercurial 3.6 introduced cgNunpacker.apply().
# Before that, there was changegroup.addchangegroup().
# Before that, there was localrepository.addchangegroup().
if hasattr(cg, 'apply'):
cg.apply(self, 'bundleclone', url)
elif hasattr(changegroup, 'addchangegroup'):
changegroup.addchangegroup(self, cg, 'bundleclone', url)
else:
self.addchangegroup(cg, 'bundleclone', url)
self.ui.status(_('finishing applying bundle; pulling\n'))
# Maintain compatibility with Mercurial 2.x.
if exchange:
return exchange.pull(self, remote, heads=heads)
else:
return self.pull(remote, heads=heads)
except (urllib2.HTTPError, urllib2.URLError) as e:
if isinstance(e, urllib2.HTTPError):
msg = _('HTTP error fetching bundle: %s') % str(e)
else:
msg = _('error fetching bundle: %s') % e.reason
# Don't fall back to regular clone unless explicitly told to.
if not self.ui.configbool('bundleclone', 'fallbackonerror', False):
raise util.Abort(msg, hint=_('consider contacting the '
'server operator if this error persists'))
self.ui.warn(msg + '\n')
self.ui.warn(_('falling back to normal clone\n'))
return super(bundleclonerepo, self).clone(remote, heads=heads,
stream=stream)
def _docheckout(ui, url, dest, upstream, revision, branch, purge, sharebase,
networkattemptlimit, networkattempts=None):
if not networkattempts:
networkattempts = [1]
def callself():
return _docheckout(ui, url, dest, upstream, revision, branch, purge,
sharebase, networkattemptlimit, networkattempts)
ui.write('ensuring %s@%s is available at %s\n' % (url, revision or branch,
dest))
destvfs = scmutil.vfs(dest, audit=False, realpath=True)
if destvfs.exists() and not destvfs.exists('.hg'):
raise error.Abort('destination exists but no .hg directory')
# Require checkouts to be tied to shared storage because efficiency.
if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
ui.warn('(destination is not shared; deleting)\n')
destvfs.rmtree(forcibly=True)
# Verify the shared path exists and is using modern pooled storage.
if destvfs.exists('.hg/sharedpath'):
storepath = destvfs.read('.hg/sharedpath').strip()
ui.write('(existing repository shared store: %s)\n' % storepath)
if not os.path.exists(storepath):
ui.warn('(shared store does not exist; deleting)\n')
destvfs.rmtree(forcibly=True)
elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
ui.warn('(shared store does not belong to pooled storage; '
'deleting to improve efficiency)\n')
destvfs.rmtree(forcibly=True)
# FUTURE when we require generaldelta, this is where we can check
# for that.
def deletesharedstore():
storepath = destvfs.read('.hg/sharedpath').strip()
if storepath.endswith('.hg'):
storepath = os.path.dirname(storepath)
storevfs = scmutil.vfs(storepath, audit=False)
storevfs.rmtree(forcibly=True)
def handlerepoerror(e):
if e.message == _('abandoned transaction found'):
ui.warn('(abandoned transaction found; trying to recover)\n')
repo = hg.repository(ui, dest)
if not repo.recover():
ui.warn('(could not recover repo state; '
'deleting shared store)\n')
deletesharedstore()
ui.warn('(attempting checkout from beginning)\n')
return callself()
raise
# At this point we either have an existing working directory using
# shared, pooled storage or we have nothing.
def handlenetworkfailure():
if networkattempts[0] >= networkattemptlimit:
raise error.Abort('reached maximum number of network attempts; '
'giving up\n')
ui.warn('(retrying after network failure on attempt %d of %d)\n' %
(networkattempts[0], networkattemptlimit))
# Do a backoff on retries to mitigate the thundering herd
# problem. This is an exponential backoff with a multipler
# plus random jitter thrown in for good measure.
# With the default settings, backoffs will be:
# 1) 2.5 - 6.5
# 2) 5.5 - 9.5
# 3) 11.5 - 15.5
backoff = (2 ** networkattempts[0] - 1) * 1.5
jittermin = ui.configint('robustcheckout', 'retryjittermin', 1000)
jittermax = ui.configint('robustcheckout', 'retryjittermax', 5000)
backoff += float(random.randint(jittermin, jittermax)) / 1000.0
ui.warn('(waiting %.2fs before retry)\n' % backoff)
time.sleep(backoff)
networkattempts[0] += 1
def handlepullerror(e):
"""Handle an exception raised during a pull.
Returns True if caller should call ``callself()`` to retry.
"""
if isinstance(e, error.Abort):
if e.args[0] == _('repository is unrelated'):
ui.warn('(repository is unrelated; deleting)\n')
destvfs.rmtree(forcibly=True)
return True
elif e.args[0].startswith(_('stream ended unexpectedly')):
ui.warn('%s\n' % e.args[0])
# Will raise if failure limit reached.
handlenetworkfailure()
return True
elif isinstance(e, urllib2.URLError):
if isinstance(e.reason, socket.error):
ui.warn('socket error: %s\n' % e.reason)
handlenetworkfailure()
return True
return False
created = False
if not destvfs.exists():
# Ensure parent directories of destination exist.
# Mercurial 3.8 removed ensuredirs and made makedirs race safe.
if util.safehasattr(util, 'ensuredirs'):
makedirs = util.ensuredirs
else:
makedirs = util.makedirs
makedirs(os.path.dirname(destvfs.base), notindexed=True)
makedirs(sharebase, notindexed=True)
if upstream:
ui.write('(cloning from upstream repo %s)\n' % upstream)
cloneurl = upstream or url
try:
res = hg.clone(ui, {}, cloneurl, dest=dest, update=False,
shareopts={'pool': sharebase, 'mode': 'identity'})
except (error.Abort, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
deletesharedstore()
return callself()
# TODO retry here.
if res is None:
raise error.Abort('clone failed')
# Verify it is using shared pool storage.
if not destvfs.exists('.hg/sharedpath'):
raise error.Abort('clone did not create a shared repo')
created = True
# The destination .hg directory should exist. Now make sure we have the
# wanted revision.
repo = hg.repository(ui, dest)
# We only pull if we are using symbolic names or the requested revision
# doesn't exist.
havewantedrev = False
if revision and revision in repo:
ctx = repo[revision]
if not ctx.hex().startswith(revision):
raise error.Abort('--revision argument is ambiguous',
hint='must be the first 12+ characters of a '
'SHA-1 fragment')
checkoutrevision = ctx.hex()
havewantedrev = True
if not havewantedrev:
ui.write('(pulling to obtain %s)\n' % (revision or branch,))
remote = None
try:
remote = hg.peer(repo, {}, url)
pullrevs = [remote.lookup(revision or branch)]
checkoutrevision = hex(pullrevs[0])
if branch:
ui.warn('(remote resolved %s to %s; '
'result is not deterministic)\n' %
(branch, checkoutrevision))
if checkoutrevision in repo:
ui.warn('(revision already present locally; not pulling)\n')
else:
pullop = exchange.pull(repo, remote, heads=pullrevs)
if not pullop.rheads:
raise error.Abort('unable to pull requested revision')
except (error.Abort, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
deletesharedstore()
return callself()
finally:
if remote:
remote.close()
# Now we should have the wanted revision in the store. Perform
# working directory manipulation.
# Purge if requested. We purge before update because this way we're
# guaranteed to not have conflicts on `hg update`.
if purge and not created:
ui.write('(purging working directory)\n')
purgeext = extensions.find('purge')
if purgeext.purge(ui, repo, all=True, abort_on_err=True,
# The function expects all arguments to be
# defined.
**{'print': None, 'print0': None, 'dirs': None,
'files': None}):
raise error.Abort('error purging')
# Update the working directory.
if commands.update(ui, repo, rev=checkoutrevision, clean=True):
raise error.Abort('error updating')
ui.write('updated to %s\n' % checkoutrevision)
return None
def apply(self, repo, source, revmap, merges, opts={}):
'''apply the revisions in revmap one by one in revision order'''
revs = sorted(revmap)
p1, p2 = repo.dirstate.parents()
pulls = []
diffopts = patch.difffeatureopts(self.ui, opts)
diffopts.git = True
lock = wlock = tr = None
try:
wlock = repo.wlock()
lock = repo.lock()
tr = repo.transaction('transplant')
for rev in revs:
node = revmap[rev]
revstr = '%s:%s' % (rev, short(node))
if self.applied(repo, node, p1):
self.ui.warn(
_('skipping already applied revision %s\n') % revstr)
continue
parents = source.changelog.parents(node)
if not (opts.get('filter') or opts.get('log')):
# If the changeset parent is the same as the
# wdir's parent, just pull it.
if parents[0] == p1:
pulls.append(node)
p1 = node
continue
if pulls:
if source != repo:
exchange.pull(repo, source.peer(), heads=pulls)
merge.update(repo, pulls[-1], False, False, None)
p1, p2 = repo.dirstate.parents()
pulls = []
domerge = False
if node in merges:
# pulling all the merge revs at once would mean we
# couldn't transplant after the latest even if
# transplants before them fail.
domerge = True
if not hasnode(repo, node):
exchange.pull(repo, source.peer(), heads=[node])
skipmerge = False
if parents[1] != revlog.nullid:
if not opts.get('parent'):
self.ui.note(
_('skipping merge changeset %s:%s\n') %
(rev, short(node)))
skipmerge = True
else:
parent = source.lookup(opts['parent'])
if parent not in parents:
raise util.Abort(
_('%s is not a parent of %s') %
(short(parent), short(node)))
else:
parent = parents[0]
if skipmerge:
patchfile = None
else:
fd, patchfile = tempfile.mkstemp(prefix='hg-transplant-')
fp = os.fdopen(fd, 'w')
gen = patch.diff(source, parent, node, opts=diffopts)
for chunk in gen:
fp.write(chunk)
fp.close()
del revmap[rev]
if patchfile or domerge:
try:
try:
n = self.applyone(repo,
node,
source.changelog.read(node),
patchfile,
merge=domerge,
log=opts.get('log'),
filter=opts.get('filter'))
except TransplantError:
# Do not rollback, it is up to the user to
# fix the merge or cancel everything
tr.close()
raise
if n and domerge:
self.ui.status(
_('%s merged at %s\n') % (revstr, short(n)))
elif n:
self.ui.status(
_('%s transplanted to %s\n') %
(short(node), short(n)))
finally:
if patchfile:
os.unlink(patchfile)
tr.close()
if pulls:
exchange.pull(repo, source.peer(), heads=pulls)
merge.update(repo, pulls[-1], False, False, None)
finally:
self.saveseries(revmap, merges)
self.transplants.write()
if tr:
tr.release()
lock.release()
wlock.release()
def _docheckout(ui, url, dest, upstream, revision, branch, purge, sharebase):
def callself():
return _docheckout(ui, url, dest, upstream, revision, branch, purge,
sharebase)
ui.write('ensuring %s@%s is available at %s\n' % (url, revision or branch,
dest))
destvfs = scmutil.vfs(dest, audit=False, realpath=True)
if destvfs.exists() and not destvfs.exists('.hg'):
raise error.Abort('destination exists but no .hg directory')
# Require checkouts to be tied to shared storage because efficiency.
if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
ui.warn('(destination is not shared; deleting)\n')
destvfs.rmtree(forcibly=True)
# Verify the shared path exists and is using modern pooled storage.
if destvfs.exists('.hg/sharedpath'):
storepath = destvfs.read('.hg/sharedpath').strip()
ui.write('(existing repository shared store: %s)\n' % storepath)
if not os.path.exists(storepath):
ui.warn('(shared store does not exist; deleting)\n')
destvfs.rmtree(forcibly=True)
elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
ui.warn('(shared store does not belong to pooled storage; '
'deleting to improve efficiency)\n')
destvfs.rmtree(forcibly=True)
# FUTURE when we require generaldelta, this is where we can check
# for that.
def deletesharedstore():
storepath = destvfs.read('.hg/sharedpath').strip()
if storepath.endswith('.hg'):
storepath = os.path.dirname(storepath)
storevfs = scmutil.vfs(storepath, audit=False)
storevfs.rmtree(forcibly=True)
def handlerepoerror(e):
if e.message == _('abandoned transaction found'):
ui.warn('(abandoned transaction found; trying to recover)\n')
repo = hg.repository(ui, dest)
if not repo.recover():
ui.warn('(could not recover repo state; '
'deleting shared store)\n')
deletesharedstore()
ui.warn('(attempting checkout from beginning)\n')
return callself()
raise
# At this point we either have an existing working directory using
# shared, pooled storage or we have nothing.
created = False
if not destvfs.exists():
# Ensure parent directories of destination exist.
# Mercurial 3.8 removed ensuredirs and made makedirs race safe.
if util.safehasattr(util, 'ensuredirs'):
makedirs = util.ensuredirs
else:
makedirs = util.makedirs
makedirs(os.path.dirname(destvfs.base), notindexed=True)
makedirs(sharebase, notindexed=True)
if upstream:
ui.write('(cloning from upstream repo %s)\n' % upstream)
cloneurl = upstream or url
try:
res = hg.clone(ui, {}, cloneurl, dest=dest, update=False,
shareopts={'pool': sharebase, 'mode': 'identity'})
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
deletesharedstore()
return callself()
# TODO retry here.
if res is None:
raise error.Abort('clone failed')
# Verify it is using shared pool storage.
if not destvfs.exists('.hg/sharedpath'):
raise error.Abort('clone did not create a shared repo')
created = True
# The destination .hg directory should exist. Now make sure we have the
# wanted revision.
repo = hg.repository(ui, dest)
# We only pull if we are using symbolic names or the requested revision
# doesn't exist.
havewantedrev = False
if revision and revision in repo:
ctx = repo[revision]
if not ctx.hex().startswith(revision):
raise error.Abort('--revision argument is ambiguous',
hint='must be the first 12+ characters of a '
'SHA-1 fragment')
havewantedrev = True
if not havewantedrev:
ui.write('(pulling to obtain %s)\n' % (revision or branch,))
try:
remote = hg.peer(repo, {}, url)
pullrevs = [remote.lookup(revision or branch)]
pullop = exchange.pull(repo, remote, heads=pullrevs)
if not pullop.rheads:
raise error.Abort('unable to pull requested revision')
except error.Abort as e:
if e.message == _('repository is unrelated'):
ui.warn('(repository is unrelated; deleting)\n')
destvfs.rmtree(forcibly=True)
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
deletesharedstore()
return callself()
finally:
remote.close()
# Now we should have the wanted revision in the store. Perform
# working directory manipulation.
# Purge if requested. We purge before update because this way we're
# guaranteed to not have conflicts on `hg update`.
if purge and not created:
ui.write('(purging working directory)\n')
purgeext = extensions.find('purge')
if purgeext.purge(ui, repo, all=True, abort_on_err=True,
# The function expects all arguments to be
# defined.
**{'print': None, 'print0': None, 'dirs': None,
'files': None}):
raise error.Abort('error purging')
# Update the working directory.
if commands.update(ui, repo, rev=revision or branch, clean=True):
raise error.Abort('error updating')
ctx = repo[revision or branch]
ui.write('updated to %s\n' % ctx.hex())
return None
def _widen(ui, repo, remote, commoninc, oldincludes, oldexcludes,
newincludes, newexcludes):
newmatch = narrowspec.match(repo.root, newincludes, newexcludes)
# for now we assume that if a server has ellipses enabled, we will be
# exchanging ellipses nodes. In future we should add ellipses as a client
# side requirement (maybe) to distinguish a client is shallow or not and
# then send that information to server whether we want ellipses or not.
# Theoretically a non-ellipses repo should be able to use narrow
# functionality from an ellipses enabled server
ellipsesremote = wireprototypes.ELLIPSESCAP in remote.capabilities()
def pullbundle2extraprepare_widen(orig, pullop, kwargs):
orig(pullop, kwargs)
# The old{in,ex}cludepats have already been set by orig()
kwargs['includepats'] = newincludes
kwargs['excludepats'] = newexcludes
wrappedextraprepare = extensions.wrappedfunction(exchange,
'_pullbundle2extraprepare', pullbundle2extraprepare_widen)
# define a function that narrowbundle2 can call after creating the
# backup bundle, but before applying the bundle from the server
def setnewnarrowpats():
repo.setnarrowpats(newincludes, newexcludes)
repo.setnewnarrowpats = setnewnarrowpats
# silence the devel-warning of applying an empty changegroup
overrides = {('devel', 'all-warnings'): False}
with ui.uninterruptable():
common = commoninc[0]
if ellipsesremote:
ds = repo.dirstate
p1, p2 = ds.p1(), ds.p2()
with ds.parentchange():
ds.setparents(node.nullid, node.nullid)
with wrappedextraprepare,\
repo.ui.configoverride(overrides, 'widen'):
exchange.pull(repo, remote, heads=common)
with ds.parentchange():
ds.setparents(p1, p2)
else:
with remote.commandexecutor() as e:
bundle = e.callcommand('narrow_widen', {
'oldincludes': oldincludes,
'oldexcludes': oldexcludes,
'newincludes': newincludes,
'newexcludes': newexcludes,
'cgversion': '03',
'commonheads': common,
'known': [],
'ellipses': False,
}).result()
with repo.transaction('widening') as tr,\
repo.ui.configoverride(overrides, 'widen'):
tgetter = lambda: tr
bundle2.processbundle(repo, bundle,
transactiongetter=tgetter)
repo.setnewnarrowpats()
actions = {k: [] for k in 'a am f g cd dc r dm dg m e k p pr'.split()}
addgaction = actions['g'].append
mf = repo['.'].manifest().matches(newmatch)
for f, fn in mf.iteritems():
if f not in repo.dirstate:
addgaction((f, (mf.flags(f), False),
"add from widened narrow clone"))
merge.applyupdates(repo, actions, wctx=repo[None],
mctx=repo['.'], overwrite=False)
merge.recordupdates(repo, actions, branchmerge=False)
def _docheckout(ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
networkattemptlimit,
networkattempts=None):
if not networkattempts:
networkattempts = [1]
def callself():
return _docheckout(ui, url, dest, upstream, revision, branch, purge,
sharebase, networkattemptlimit, networkattempts)
ui.write('ensuring %s@%s is available at %s\n' %
(url, revision or branch, dest))
# We assume that we're the only process on the machine touching the
# repository paths that we were told to use. This means our recovery
# scenario when things aren't "right" is to just nuke things and start
# from scratch. This is easier to implement than verifying the state
# of the data and attempting recovery. And in some scenarios (such as
# potential repo corruption), it is probably faster, since verifying
# repos can take a while.
destvfs = getvfs()(dest, audit=False, realpath=True)
def deletesharedstore(path=None):
storepath = path or destvfs.read('.hg/sharedpath').strip()
if storepath.endswith('.hg'):
storepath = os.path.dirname(storepath)
storevfs = getvfs()(storepath, audit=False)
storevfs.rmtree(forcibly=True)
if destvfs.exists() and not destvfs.exists('.hg'):
raise error.Abort('destination exists but no .hg directory')
# Require checkouts to be tied to shared storage because efficiency.
if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
ui.warn('(destination is not shared; deleting)\n')
destvfs.rmtree(forcibly=True)
# Verify the shared path exists and is using modern pooled storage.
if destvfs.exists('.hg/sharedpath'):
storepath = destvfs.read('.hg/sharedpath').strip()
ui.write('(existing repository shared store: %s)\n' % storepath)
if not os.path.exists(storepath):
ui.warn('(shared store does not exist; deleting destination)\n')
destvfs.rmtree(forcibly=True)
elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
ui.warn('(shared store does not belong to pooled storage; '
'deleting destination to improve efficiency)\n')
destvfs.rmtree(forcibly=True)
storevfs = getvfs()(storepath, audit=False)
if storevfs.isfileorlink('store/lock'):
ui.warn('(shared store has an active lock; assuming it is left '
'over from a previous process and that the store is '
'corrupt; deleting store and destination just to be '
'sure)\n')
destvfs.rmtree(forcibly=True)
deletesharedstore(storepath)
# FUTURE when we require generaldelta, this is where we can check
# for that.
if destvfs.isfileorlink('.hg/wlock'):
ui.warn('(dest has an active working directory lock; assuming it is '
'left over from a previous process and that the destination '
'is corrupt; deleting it just to be sure)\n')
destvfs.rmtree(forcibly=True)
def handlerepoerror(e):
if e.message == _('abandoned transaction found'):
ui.warn('(abandoned transaction found; trying to recover)\n')
repo = hg.repository(ui, dest)
if not repo.recover():
ui.warn('(could not recover repo state; '
'deleting shared store)\n')
deletesharedstore()
ui.warn('(attempting checkout from beginning)\n')
return callself()
raise
# At this point we either have an existing working directory using
# shared, pooled storage or we have nothing.
def handlenetworkfailure():
if networkattempts[0] >= networkattemptlimit:
raise error.Abort('reached maximum number of network attempts; '
'giving up\n')
ui.warn('(retrying after network failure on attempt %d of %d)\n' %
(networkattempts[0], networkattemptlimit))
# Do a backoff on retries to mitigate the thundering herd
# problem. This is an exponential backoff with a multipler
# plus random jitter thrown in for good measure.
# With the default settings, backoffs will be:
# 1) 2.5 - 6.5
# 2) 5.5 - 9.5
# 3) 11.5 - 15.5
backoff = (2**networkattempts[0] - 1) * 1.5
jittermin = ui.configint('robustcheckout', 'retryjittermin', 1000)
jittermax = ui.configint('robustcheckout', 'retryjittermax', 5000)
backoff += float(random.randint(jittermin, jittermax)) / 1000.0
ui.warn('(waiting %.2fs before retry)\n' % backoff)
time.sleep(backoff)
networkattempts[0] += 1
def handlepullerror(e):
"""Handle an exception raised during a pull.
Returns True if caller should call ``callself()`` to retry.
"""
if isinstance(e, error.Abort):
if e.args[0] == _('repository is unrelated'):
ui.warn('(repository is unrelated; deleting)\n')
destvfs.rmtree(forcibly=True)
return True
elif e.args[0].startswith(_('stream ended unexpectedly')):
ui.warn('%s\n' % e.args[0])
# Will raise if failure limit reached.
handlenetworkfailure()
return True
elif isinstance(e, ssl.SSLError):
# Assume all SSL errors are due to the network, as Mercurial
# should convert non-transport errors like cert validation failures
# to error.Abort.
ui.warn('ssl error: %s\n' % e)
handlenetworkfailure()
return True
elif isinstance(e, urllib2.URLError):
if isinstance(e.reason, socket.error):
ui.warn('socket error: %s\n' % e.reason)
handlenetworkfailure()
return True
return False
created = False
if not destvfs.exists():
# Ensure parent directories of destination exist.
# Mercurial 3.8 removed ensuredirs and made makedirs race safe.
if util.safehasattr(util, 'ensuredirs'):
makedirs = util.ensuredirs
else:
makedirs = util.makedirs
makedirs(os.path.dirname(destvfs.base), notindexed=True)
makedirs(sharebase, notindexed=True)
if upstream:
ui.write('(cloning from upstream repo %s)\n' % upstream)
cloneurl = upstream or url
try:
res = hg.clone(ui, {},
cloneurl,
dest=dest,
update=False,
shareopts={
'pool': sharebase,
'mode': 'identity'
})
except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' %
e.message)
deletesharedstore()
return callself()
# TODO retry here.
if res is None:
raise error.Abort('clone failed')
# Verify it is using shared pool storage.
if not destvfs.exists('.hg/sharedpath'):
raise error.Abort('clone did not create a shared repo')
created = True
# The destination .hg directory should exist. Now make sure we have the
# wanted revision.
repo = hg.repository(ui, dest)
# We only pull if we are using symbolic names or the requested revision
# doesn't exist.
havewantedrev = False
if revision and revision in repo:
ctx = repo[revision]
if not ctx.hex().startswith(revision):
raise error.Abort('--revision argument is ambiguous',
hint='must be the first 12+ characters of a '
'SHA-1 fragment')
checkoutrevision = ctx.hex()
havewantedrev = True
if not havewantedrev:
ui.write('(pulling to obtain %s)\n' % (revision or branch, ))
remote = None
try:
remote = hg.peer(repo, {}, url)
pullrevs = [remote.lookup(revision or branch)]
checkoutrevision = hex(pullrevs[0])
if branch:
ui.warn('(remote resolved %s to %s; '
'result is not deterministic)\n' %
(branch, checkoutrevision))
if checkoutrevision in repo:
ui.warn('(revision already present locally; not pulling)\n')
else:
pullop = exchange.pull(repo, remote, heads=pullrevs)
if not pullop.rheads:
raise error.Abort('unable to pull requested revision')
except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' %
e.message)
deletesharedstore()
return callself()
finally:
if remote:
remote.close()
# Now we should have the wanted revision in the store. Perform
# working directory manipulation.
# Purge if requested. We purge before update because this way we're
# guaranteed to not have conflicts on `hg update`.
if purge and not created:
ui.write('(purging working directory)\n')
purgeext = extensions.find('purge')
if purgeext.purge(
ui,
repo,
all=True,
abort_on_err=True,
# The function expects all arguments to be
# defined.
**{
'print': None,
'print0': None,
'dirs': None,
'files': None
}):
raise error.Abort('error purging')
# Update the working directory.
if commands.update(ui, repo, rev=checkoutrevision, clean=True):
raise error.Abort('error updating')
ui.write('updated to %s\n' % checkoutrevision)
return None
def _do_case(self, name, layout):
subdir = test_util.subdir.get(name, '')
single = layout == 'single'
u = ui.ui()
config = {}
if layout == 'custom':
for branch, path in test_util.custom.get(name, {}).iteritems():
config['hgsubversionbranch.%s' % branch] = path
u.setconfig('hgsubversionbranch', branch, path)
repo, repo_path = self.load_and_fetch(name, subdir=subdir, layout=layout)
assert test_util.repolen(self.repo) > 0
wc2_path = self.wc_path + '_clone'
src, dest = test_util.hgclone(u, self.wc_path, wc2_path, update=False)
src = test_util.getlocalpeer(src)
dest = test_util.getlocalpeer(dest)
# insert a wrapper that prevents calling changectx.children()
def failfn(orig, ctx):
self.fail('calling %s is forbidden; it can cause massive slowdowns '
'when rebuilding large repositories' % orig)
origchildren = getattr(context.changectx, 'children')
extensions.wrapfunction(context.changectx, 'children', failfn)
try:
svncommands.rebuildmeta(u, dest,
args=[test_util.fileurl(repo_path +
subdir), ])
finally:
# remove the wrapper
context.changectx.children = origchildren
self._run_assertions(name, single, src, dest, u)
wc3_path = self.wc_path + '_partial'
src, dest = test_util.hgclone(u,
self.wc_path,
wc3_path,
update=False,
rev=[0])
srcrepo = test_util.getlocalpeer(src)
dest = test_util.getlocalpeer(dest)
# insert a wrapper that prevents calling changectx.children()
extensions.wrapfunction(context.changectx, 'children', failfn)
try:
svncommands.rebuildmeta(u, dest,
args=[test_util.fileurl(repo_path +
subdir), ])
finally:
# remove the wrapper
context.changectx.children = origchildren
if hgutil.safehasattr(localrepo.localrepository, 'pull'):
dest.pull(src)
else:
# Mercurial >= 3.2
from mercurial import exchange
exchange.pull(dest, src)
# insert a wrapper that prevents calling changectx.children()
extensions.wrapfunction(context.changectx, 'children', failfn)
try:
svncommands.updatemeta(u, dest,
args=[test_util.fileurl(repo_path +
subdir), ])
finally:
# remove the wrapper
context.changectx.children = origchildren
self._run_assertions(name, single, srcrepo, dest, u)
def fetch(ui, repo, source='default', **opts):
'''pull changes from a remote repository, merge new changes if needed.
This finds all changes from the repository at the specified path
or URL and adds them to the local repository.
If the pulled changes add a new branch head, the head is
automatically merged, and the result of the merge is committed.
Otherwise, the working directory is updated to include the new
changes.
When a merge is needed, the working directory is first updated to
the newly pulled changes. Local changes are then merged into the
pulled changes. To switch the merge order, use --switch-parent.
See :hg:`help dates` for a list of formats valid for -d/--date.
Returns 0 on success.
'''
date = opts.get('date')
if date:
opts['date'] = util.parsedate(date)
parent, _p2 = repo.dirstate.parents()
branch = repo.dirstate.branch()
try:
branchnode = repo.branchtip(branch)
except error.RepoLookupError:
branchnode = None
if parent != branchnode:
raise util.Abort(_('working dir not at branch tip '
'(use "hg update" to check out branch tip)'))
wlock = lock = None
try:
wlock = repo.wlock()
lock = repo.lock()
cmdutil.bailifchanged(repo)
bheads = repo.branchheads(branch)
bheads = [head for head in bheads if len(repo[head].children()) == 0]
if len(bheads) > 1:
raise util.Abort(_('multiple heads in this branch '
'(use "hg heads ." and "hg merge" to merge)'))
other = hg.peer(repo, opts, ui.expandpath(source))
ui.status(_('pulling from %s\n') %
util.hidepassword(ui.expandpath(source)))
revs = None
if opts['rev']:
try:
revs = [other.lookup(rev) for rev in opts['rev']]
except error.CapabilityError:
err = _("other repository doesn't support revision lookup, "
"so a rev cannot be specified.")
raise util.Abort(err)
# Are there any changes at all?
modheads = exchange.pull(repo, other, heads=revs).cgresult
if modheads == 0:
return 0
# Is this a simple fast-forward along the current branch?
newheads = repo.branchheads(branch)
newchildren = repo.changelog.nodesbetween([parent], newheads)[2]
if len(newheads) == 1 and len(newchildren):
if newchildren[0] != parent:
return hg.update(repo, newchildren[0])
else:
return 0
# Are there more than one additional branch heads?
newchildren = [n for n in newchildren if n != parent]
newparent = parent
if newchildren:
newparent = newchildren[0]
hg.clean(repo, newparent)
newheads = [n for n in newheads if n != newparent]
if len(newheads) > 1:
ui.status(_('not merging with %d other new branch heads '
'(use "hg heads ." and "hg merge" to merge them)\n') %
(len(newheads) - 1))
return 1
if not newheads:
return 0
# Otherwise, let's merge.
err = False
if newheads:
# By default, we consider the repository we're pulling
# *from* as authoritative, so we merge our changes into
# theirs.
if opts['switch_parent']:
firstparent, secondparent = newparent, newheads[0]
else:
firstparent, secondparent = newheads[0], newparent
ui.status(_('updating to %d:%s\n') %
(repo.changelog.rev(firstparent),
short(firstparent)))
hg.clean(repo, firstparent)
ui.status(_('merging with %d:%s\n') %
(repo.changelog.rev(secondparent), short(secondparent)))
err = hg.merge(repo, secondparent, remind=False)
if not err:
# we don't translate commit messages
message = (cmdutil.logmessage(ui, opts) or
('Automated merge with %s' %
util.removeauth(other.url())))
editopt = opts.get('edit') or opts.get('force_editor')
editor = cmdutil.getcommiteditor(edit=editopt, editform='fetch')
n = repo.commit(message, opts['user'], opts['date'], editor=editor)
ui.status(_('new changeset %d:%s merges remote changes '
'with local\n') % (repo.changelog.rev(n),
short(n)))
return err
finally:
release(lock, wlock)
def _docheckout(
ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
optimes,
behaviors,
networkattemptlimit,
networkattempts=None,
sparse_profile=None,
noupdate=False,
):
if not networkattempts:
networkattempts = [1]
def callself():
return _docheckout(
ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
optimes,
behaviors,
networkattemptlimit,
networkattempts=networkattempts,
sparse_profile=sparse_profile,
noupdate=noupdate,
)
@contextlib.contextmanager
def timeit(op, behavior):
behaviors.add(behavior)
errored = False
try:
start = time.time()
yield
except Exception:
errored = True
raise
finally:
elapsed = time.time() - start
if errored:
op += "_errored"
optimes.append((op, elapsed))
ui.write(b"ensuring %s@%s is available at %s\n" % (url, revision or branch, dest))
# We assume that we're the only process on the machine touching the
# repository paths that we were told to use. This means our recovery
# scenario when things aren't "right" is to just nuke things and start
# from scratch. This is easier to implement than verifying the state
# of the data and attempting recovery. And in some scenarios (such as
# potential repo corruption), it is probably faster, since verifying
# repos can take a while.
destvfs = vfs.vfs(dest, audit=False, realpath=True)
def deletesharedstore(path=None):
storepath = path or destvfs.read(b".hg/sharedpath").strip()
if storepath.endswith(b".hg"):
storepath = os.path.dirname(storepath)
storevfs = vfs.vfs(storepath, audit=False)
storevfs.rmtree(forcibly=True)
if destvfs.exists() and not destvfs.exists(b".hg"):
raise error.Abort(b"destination exists but no .hg directory")
# Refuse to enable sparse checkouts on existing checkouts. The reasoning
# here is that another consumer of this repo may not be sparse aware. If we
# enabled sparse, we would lock them out.
if destvfs.exists() and sparse_profile and not destvfs.exists(b".hg/sparse"):
raise error.Abort(
b"cannot enable sparse profile on existing " b"non-sparse checkout",
hint=b"use a separate working directory to use sparse",
)
# And the other direction for symmetry.
if not sparse_profile and destvfs.exists(b".hg/sparse"):
raise error.Abort(
b"cannot use non-sparse checkout on existing sparse " b"checkout",
hint=b"use a separate working directory to use sparse",
)
# Require checkouts to be tied to shared storage because efficiency.
if destvfs.exists(b".hg") and not destvfs.exists(b".hg/sharedpath"):
ui.warn(b"(destination is not shared; deleting)\n")
with timeit("remove_unshared_dest", "remove-wdir"):
destvfs.rmtree(forcibly=True)
# Verify the shared path exists and is using modern pooled storage.
if destvfs.exists(b".hg/sharedpath"):
storepath = destvfs.read(b".hg/sharedpath").strip()
ui.write(b"(existing repository shared store: %s)\n" % storepath)
if not os.path.exists(storepath):
ui.warn(b"(shared store does not exist; deleting destination)\n")
with timeit("removed_missing_shared_store", "remove-wdir"):
destvfs.rmtree(forcibly=True)
elif not re.search(b"[a-f0-9]{40}/\.hg$", storepath.replace(b"\\", b"/")):
ui.warn(
b"(shared store does not belong to pooled storage; "
b"deleting destination to improve efficiency)\n"
)
with timeit("remove_unpooled_store", "remove-wdir"):
destvfs.rmtree(forcibly=True)
if destvfs.isfileorlink(b".hg/wlock"):
ui.warn(
b"(dest has an active working directory lock; assuming it is "
b"left over from a previous process and that the destination "
b"is corrupt; deleting it just to be sure)\n"
)
with timeit("remove_locked_wdir", "remove-wdir"):
destvfs.rmtree(forcibly=True)
def handlerepoerror(e):
if pycompat.bytestr(e) == _(b"abandoned transaction found"):
ui.warn(b"(abandoned transaction found; trying to recover)\n")
repo = hg.repository(ui, dest)
if not repo.recover():
ui.warn(b"(could not recover repo state; " b"deleting shared store)\n")
with timeit("remove_unrecovered_shared_store", "remove-store"):
deletesharedstore()
ui.warn(b"(attempting checkout from beginning)\n")
return callself()
raise
# At this point we either have an existing working directory using
# shared, pooled storage or we have nothing.
def handlenetworkfailure():
if networkattempts[0] >= networkattemptlimit:
raise error.Abort(
b"reached maximum number of network attempts; " b"giving up\n"
)
ui.warn(
b"(retrying after network failure on attempt %d of %d)\n"
% (networkattempts[0], networkattemptlimit)
)
# Do a backoff on retries to mitigate the thundering herd
# problem. This is an exponential backoff with a multipler
# plus random jitter thrown in for good measure.
# With the default settings, backoffs will be:
# 1) 2.5 - 6.5
# 2) 5.5 - 9.5
# 3) 11.5 - 15.5
backoff = (2 ** networkattempts[0] - 1) * 1.5
jittermin = ui.configint(b"robustcheckout", b"retryjittermin", 1000)
jittermax = ui.configint(b"robustcheckout", b"retryjittermax", 5000)
backoff += float(random.randint(jittermin, jittermax)) / 1000.0
ui.warn(b"(waiting %.2fs before retry)\n" % backoff)
time.sleep(backoff)
networkattempts[0] += 1
def handlepullerror(e):
"""Handle an exception raised during a pull.
Returns True if caller should call ``callself()`` to retry.
"""
if isinstance(e, error.Abort):
if e.args[0] == _(b"repository is unrelated"):
ui.warn(b"(repository is unrelated; deleting)\n")
destvfs.rmtree(forcibly=True)
return True
elif e.args[0].startswith(_(b"stream ended unexpectedly")):
ui.warn(b"%s\n" % e.args[0])
# Will raise if failure limit reached.
handlenetworkfailure()
return True
# TODO test this branch
elif isinstance(e, error.ResponseError):
if e.args[0].startswith(_(b"unexpected response from remote server:")):
ui.warn(b"(unexpected response from remote server; retrying)\n")
destvfs.rmtree(forcibly=True)
# Will raise if failure limit reached.
handlenetworkfailure()
return True
elif isinstance(e, ssl.SSLError):
# Assume all SSL errors are due to the network, as Mercurial
# should convert non-transport errors like cert validation failures
# to error.Abort.
ui.warn(b"ssl error: %s\n" % e)
handlenetworkfailure()
return True
elif isinstance(e, urllibcompat.urlerr.urlerror):
if isinstance(e.reason, socket.error):
ui.warn(b"socket error: %s\n" % pycompat.bytestr(e.reason))
handlenetworkfailure()
return True
else:
ui.warn(
b"unhandled URLError; reason type: %s; value: %s\n"
% (e.reason.__class__.__name__, e.reason)
)
else:
ui.warn(
b"unhandled exception during network operation; type: %s; "
b"value: %s\n" % (e.__class__.__name__, e)
)
return False
# Perform sanity checking of store. We may or may not know the path to the
# local store. It depends if we have an existing destvfs pointing to a
# share. To ensure we always find a local store, perform the same logic
# that Mercurial's pooled storage does to resolve the local store path.
cloneurl = upstream or url
try:
clonepeer = hg.peer(ui, {}, cloneurl)
rootnode = peerlookup(clonepeer, b"0")
except error.RepoLookupError:
raise error.Abort(b"unable to resolve root revision from clone " b"source")
except (error.Abort, ssl.SSLError, urllibcompat.urlerr.urlerror) as e:
if handlepullerror(e):
return callself()
raise
if rootnode == nullid:
raise error.Abort(b"source repo appears to be empty")
storepath = os.path.join(sharebase, hex(rootnode))
storevfs = vfs.vfs(storepath, audit=False)
if storevfs.isfileorlink(b".hg/store/lock"):
ui.warn(
b"(shared store has an active lock; assuming it is left "
b"over from a previous process and that the store is "
b"corrupt; deleting store and destination just to be "
b"sure)\n"
)
if destvfs.exists():
with timeit("remove_dest_active_lock", "remove-wdir"):
destvfs.rmtree(forcibly=True)
with timeit("remove_shared_store_active_lock", "remove-store"):
storevfs.rmtree(forcibly=True)
if storevfs.exists() and not storevfs.exists(b".hg/requires"):
ui.warn(
b"(shared store missing requires file; this is a really "
b"odd failure; deleting store and destination)\n"
)
if destvfs.exists():
with timeit("remove_dest_no_requires", "remove-wdir"):
destvfs.rmtree(forcibly=True)
with timeit("remove_shared_store_no_requires", "remove-store"):
storevfs.rmtree(forcibly=True)
if storevfs.exists(b".hg/requires"):
requires = set(storevfs.read(b".hg/requires").splitlines())
# FUTURE when we require generaldelta, this is where we can check
# for that.
required = {b"dotencode", b"fncache"}
missing = required - requires
if missing:
ui.warn(
b"(shared store missing requirements: %s; deleting "
b"store and destination to ensure optimal behavior)\n"
% b", ".join(sorted(missing))
)
if destvfs.exists():
with timeit("remove_dest_missing_requires", "remove-wdir"):
destvfs.rmtree(forcibly=True)
with timeit("remove_shared_store_missing_requires", "remove-store"):
storevfs.rmtree(forcibly=True)
created = False
if not destvfs.exists():
# Ensure parent directories of destination exist.
# Mercurial 3.8 removed ensuredirs and made makedirs race safe.
if util.safehasattr(util, "ensuredirs"):
makedirs = util.ensuredirs
else:
makedirs = util.makedirs
makedirs(os.path.dirname(destvfs.base), notindexed=True)
makedirs(sharebase, notindexed=True)
if upstream:
ui.write(b"(cloning from upstream repo %s)\n" % upstream)
if not storevfs.exists():
behaviors.add(b"create-store")
try:
with timeit("clone", "clone"):
shareopts = {b"pool": sharebase, b"mode": b"identity"}
res = hg.clone(
ui,
{},
clonepeer,
dest=dest,
update=False,
shareopts=shareopts,
stream=True,
)
except (error.Abort, ssl.SSLError, urllibcompat.urlerr.urlerror) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn(b"(repo corruption: %s; deleting shared store)\n" % e)
with timeit("remove_shared_store_revlogerror", "remote-store"):
deletesharedstore()
return callself()
# TODO retry here.
if res is None:
raise error.Abort(b"clone failed")
# Verify it is using shared pool storage.
if not destvfs.exists(b".hg/sharedpath"):
raise error.Abort(b"clone did not create a shared repo")
created = True
# The destination .hg directory should exist. Now make sure we have the
# wanted revision.
repo = hg.repository(ui, dest)
# We only pull if we are using symbolic names or the requested revision
# doesn't exist.
havewantedrev = False
if revision:
try:
ctx = scmutil.revsingle(repo, revision)
except error.RepoLookupError:
ctx = None
if ctx:
if not ctx.hex().startswith(revision):
raise error.Abort(
b"--revision argument is ambiguous",
hint=b"must be the first 12+ characters of a " b"SHA-1 fragment",
)
checkoutrevision = ctx.hex()
havewantedrev = True
if not havewantedrev:
ui.write(b"(pulling to obtain %s)\n" % (revision or branch,))
remote = None
try:
remote = hg.peer(repo, {}, url)
pullrevs = [peerlookup(remote, revision or branch)]
checkoutrevision = hex(pullrevs[0])
if branch:
ui.warn(
b"(remote resolved %s to %s; "
b"result is not deterministic)\n" % (branch, checkoutrevision)
)
if checkoutrevision in repo:
ui.warn(b"(revision already present locally; not pulling)\n")
else:
with timeit("pull", "pull"):
pullop = exchange.pull(repo, remote, heads=pullrevs)
if not pullop.rheads:
raise error.Abort(b"unable to pull requested revision")
except (error.Abort, ssl.SSLError, urllibcompat.urlerr.urlerror) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn(b"(repo corruption: %s; deleting shared store)\n" % e)
deletesharedstore()
return callself()
finally:
if remote:
remote.close()
# Now we should have the wanted revision in the store. Perform
# working directory manipulation.
# Avoid any working directory manipulations if `-U`/`--noupdate` was passed
if noupdate:
ui.write(b"(skipping update since `-U` was passed)\n")
return None
# Purge if requested. We purge before update because this way we're
# guaranteed to not have conflicts on `hg update`.
if purge and not created:
ui.write(b"(purging working directory)\n")
purgeext = extensions.find(b"purge")
# Mercurial 4.3 doesn't purge files outside the sparse checkout.
# See https://bz.mercurial-scm.org/show_bug.cgi?id=5626. Force
# purging by monkeypatching the sparse matcher.
try:
old_sparse_fn = getattr(repo.dirstate, "_sparsematchfn", None)
if old_sparse_fn is not None:
# TRACKING hg50
# Arguments passed to `matchmod.always` were unused and have been removed
if util.versiontuple(n=2) >= (5, 0):
repo.dirstate._sparsematchfn = lambda: matchmod.always()
else:
repo.dirstate._sparsematchfn = lambda: matchmod.always(
repo.root, ""
)
with timeit("purge", "purge"):
if purgeext.purge(
ui,
repo,
all=True,
abort_on_err=True,
# The function expects all arguments to be
# defined.
**{"print": None, "print0": None, "dirs": None, "files": None}
):
raise error.Abort(b"error purging")
finally:
if old_sparse_fn is not None:
repo.dirstate._sparsematchfn = old_sparse_fn
# Update the working directory.
if repo[b"."].node() == nullid:
behaviors.add("empty-wdir")
else:
behaviors.add("populated-wdir")
if sparse_profile:
sparsemod = getsparse()
# By default, Mercurial will ignore unknown sparse profiles. This could
# lead to a full checkout. Be more strict.
try:
repo.filectx(sparse_profile, changeid=checkoutrevision).data()
except error.ManifestLookupError:
raise error.Abort(
b"sparse profile %s does not exist at revision "
b"%s" % (sparse_profile, checkoutrevision)
)
# TRACKING hg48 - parseconfig takes `action` param
if util.versiontuple(n=2) >= (4, 8):
old_config = sparsemod.parseconfig(
repo.ui, repo.vfs.tryread(b"sparse"), b"sparse"
)
else:
old_config = sparsemod.parseconfig(repo.ui, repo.vfs.tryread(b"sparse"))
old_includes, old_excludes, old_profiles = old_config
if old_profiles == {sparse_profile} and not old_includes and not old_excludes:
ui.write(
b"(sparse profile %s already set; no need to update "
b"sparse config)\n" % sparse_profile
)
else:
if old_includes or old_excludes or old_profiles:
ui.write(
b"(replacing existing sparse config with profile "
b"%s)\n" % sparse_profile
)
else:
ui.write(b"(setting sparse config to profile %s)\n" % sparse_profile)
# If doing an incremental update, this will perform two updates:
# one to change the sparse profile and another to update to the new
# revision. This is not desired. But there's not a good API in
# Mercurial to do this as one operation.
with repo.wlock(), timeit("sparse_update_config", "sparse-update-config"):
# pylint --py3k: W1636
fcounts = list(
map(
len,
sparsemod._updateconfigandrefreshwdir(
repo, [], [], [sparse_profile], force=True
),
)
)
repo.ui.status(
b"%d files added, %d files dropped, "
b"%d files conflicting\n" % tuple(fcounts)
)
ui.write(b"(sparse refresh complete)\n")
op = "update_sparse" if sparse_profile else "update"
behavior = "update-sparse" if sparse_profile else "update"
with timeit(op, behavior):
if commands.update(ui, repo, rev=checkoutrevision, clean=True):
raise error.Abort(b"error updating")
ui.write(b"updated to %s\n" % checkoutrevision)
return None
def unifyrepo(ui, settings):
"""Unify the contents of multiple source repositories using settings.
The settings file is a Mercurial config file (basically an INI file).
"""
conf = unifyconfig(settings)
# Ensure destrepo is created with generaldelta enabled.
ui.setconfig('format', 'usegeneraldelta', True)
ui.setconfig('format', 'generaldelta', True)
# Verify all source repos have the same revision 0
rev0s = set()
for source in conf.sources:
repo = hg.repository(ui, path=source['path'])
# Verify
node = repo[0].node()
if rev0s and node not in rev0s:
ui.warn('repository has different rev 0: %s\n' % source['name'])
rev0s.add(node)
# Ensure the staging repo has all changesets from the source repos.
stageui = ui.copy()
# Enable aggressive merge deltas on the stage repo to minimize manifest delta
# size. This could make delta chains very long. So we may want to institute a
# delta chain cap on the destination repo. But this will ensure the stage repo
# has the most efficient/compact representation of deltas. Pulling from this
# repo will also inherit the optimal delta, so we don't need to enable
# aggressivemergedeltas on the destination repo.
stageui.setconfig('format', 'aggressivemergedeltas', True)
stagerepo = hg.repository(stageui, path=conf.stagepath,
create=not os.path.exists(conf.stagepath))
for source in conf.sources:
path = source['path']
sourcepeer = hg.peer(ui, {}, path)
ui.write('pulling %s into %s\n' % (path, conf.stagepath))
exchange.pull(stagerepo, sourcepeer)
# Now collect all the changeset data with pushlog info.
# node -> (when, source, rev, who, pushid)
nodepushinfo = {}
pushcount = 0
allnodes = set()
# Obtain pushlog data from each source repo. We obtain data for every node
# and filter later because we want to be sure we have the earliest known
# push data for a given node.
for source in conf.sources:
sourcerepo = hg.repository(ui, path=source['path'])
pushlog = getattr(sourcerepo, 'pushlog', None)
if not pushlog:
raise error.Abort('pushlog API not available',
hint='is the pushlog extension loaded?')
index = sourcerepo.changelog.index
revnode = {}
for rev in sourcerepo:
# revlog.node() is too slow. Use the index directly.
node = index[rev][7]
revnode[rev] = node
allnodes.add(node)
noderev = {v: k for k, v in revnode.iteritems()}
localpushcount = 0
pushnodecount = 0
for pushid, who, when, nodes in pushlog.pushes():
pushcount += 1
localpushcount += 1
for node in nodes:
pushnodecount += 1
bnode = bin(node)
# There is a race between us iterating the repo and querying the
# pushlog. A new changeset could be written between when we
# obtain nodes and encounter the pushlog. So ignore pushlog
# for nodes we don't know about.
if bnode not in noderev:
ui.warn('pushlog entry for unknown node: %s; '
'possible race condition?\n' % node)
continue
rev = noderev[bnode]
if bnode not in nodepushinfo:
nodepushinfo[bnode] = (when, path, rev, who, pushid)
else:
currentwhen = nodepushinfo[bnode][0]
if when < currentwhen:
nodepushinfo[bnode] = (when, path, rev, who, pushid)
ui.write('obtained pushlog info for %d/%d revisions from %d pushes from %s\n' % (
pushnodecount, len(revnode), localpushcount, source['name']))
# Now verify that every node in the source repos has pushlog data.
missingpl = allnodes - set(nodepushinfo.keys())
if missingpl:
raise error.Abort('missing pushlog info for %d nodes\n' % len(missingpl))
# Filter out changesets we aren't aggregating.
# We also use this pass to identify which nodes to bookmark.
books = {}
sourcenodes = set()
for source in conf.sources:
sourcerepo = hg.repository(ui, path=source['path'])
cl = sourcerepo.changelog
index = cl.index
sourcerevs = sourcerepo.revs(source['pullrevs'])
sourcerevs.sort()
headrevs = set(cl.headrevs())
sourceheadrevs = headrevs & set(sourcerevs)
# We /could/ allow multiple heads from each source repo. But for now
# it is easier to limit to 1 head per source.
if len(sourceheadrevs) > 1:
raise error.Abort('%s has %d heads' % (source['name'], len(sourceheadrevs)),
hint='define pullrevs to limit what is aggregated')
for rev in cl:
if rev not in sourcerevs:
continue
node = index[rev][7]
sourcenodes.add(node)
if source['bookmark']:
books[source['bookmark']] = node
ui.write('aggregating %d/%d revisions for %d heads from %s\n' % (
len(sourcerevs), len(cl), len(sourceheadrevs), source['name']))
nodepushinfo = {k: v for k, v in nodepushinfo.iteritems() if k in sourcenodes}
ui.write('aggregating %d/%d nodes from %d original pushes\n' % (
len(nodepushinfo), len(allnodes), pushcount))
# We now have accounting for every changeset. Because pulling changesets
# is a bit time consuming, it is worthwhile to minimize the number of pull
# operations. We do this by ordering all changesets by original push time
# then emitting the minimum number of "fast forward" nodes from the tip
# of each linear range inside that list.
# (time, source, rev, user, pushid) -> node
inversenodeinfo = {v: k for k, v in nodepushinfo.iteritems()}
destui = ui.copy()
destui.setconfig('format', 'aggressivemergedeltas', True)
destui.setconfig('format', 'maxchainlen', 10000)
destrepo = hg.repository(destui, path=conf.destpath,
create=not os.path.exists(conf.destpath))
destcl = destrepo.changelog
pullpushinfo = {k: v for k, v in inversenodeinfo.iteritems() if not destcl.hasnode(v)}
ui.write('%d/%d nodes will be pulled\n' % (len(pullpushinfo), len(inversenodeinfo)))
pullnodes = list(emitfastforwardnodes(stagerepo, pullpushinfo))
unifiedpushes = list(unifypushes(inversenodeinfo))
ui.write('consolidated into %d pulls from %d unique pushes\n' % (
len(pullnodes), len(unifiedpushes)))
if not pullnodes:
ui.write('nothing to do; exiting\n')
return
stagepeer = hg.peer(ui, {}, conf.stagepath)
for node in pullnodes:
# TODO Bug 1265002 - we should update bookmarks when we pull.
# Otherwise the changesets will get replicated without a bookmark
# and any poor soul who pulls will see a nameless head.
exchange.pull(destrepo, stagepeer, heads=[node])
# For some reason there is a massive memory leak (10+ MB per
# iteration on Firefox repos) if we don't gc here.
gc.collect()
# Now that we've aggregated all the changesets in the destination repo,
# define the pushlog entries.
pushlog = getattr(destrepo, 'pushlog', None)
if not pushlog:
raise error.Abort('pushlog API not available',
hint='is the pushlog extension loaded?')
insertpushes = list(newpushes(destrepo, unifiedpushes))
ui.write('inserting %d pushlog entries\n' % len(insertpushes))
pushlog.recordpushes(insertpushes)
# Verify that pushlog time in revision order is always increasing.
destnodepushtime = {}
for pushid, who, when, nodes in destrepo.pushlog.pushes():
for node in nodes:
destnodepushtime[bin(node)] = when
destcl = destrepo.changelog
lastpushtime = 0
for rev in destrepo:
node = destcl.node(rev)
pushtime = destnodepushtime[node]
if pushtime < lastpushtime:
ui.warn('push time for %d is older than %d\n' % (rev, rev - 1))
lastpushtime = pushtime
# Write bookmarks.
ui.write('writing %d bookmarks\n' % len(books))
with destrepo.lock():
with destrepo.transaction('bookmarks') as tr:
bm = bookmarks.bmstore(destrepo)
# Mass replacing may not be the proper strategy. But it works for
# our current use case.
bm.clear()
bm.update(books)
bm.recordchange(tr)
# This is a bit hacky. Pushlog and bookmarks aren't currently replicated
# via the normal hooks mechanism because we use the low-level APIs to
# write them. So, we send a replication message to sync the entire repo.
try:
vcsr = extensions.find('vcsreplicator')
except KeyError:
raise error.Abort('vcsreplicator extension not installed; '
'pushlog and bookmarks may not be replicated properly')
vcsr.replicatecommand(destrepo.ui, destrepo)
