def __init__(self, repo, name, filetype=None):
self.repo = repo
self.name = name
self.vfs = vfsmod.vfs(repo.vfs.join(shelvedir))
self.backupvfs = vfsmod.vfs(repo.vfs.join(backupdir))
self.ui = self.repo.ui
if filetype:
self.fname = name + '.' + filetype
else:
self.fname = name
def __init__(self, repo, name, filetype=None):
self.repo = repo
self.name = name
self.vfs = vfsmod.vfs(repo.vfs.join(shelvedir))
self.backupvfs = vfsmod.vfs(repo.vfs.join(backupdir))
self.ui = self.repo.ui
if filetype:
self.fname = name + '.' + filetype
else:
self.fname = name
def __init__(self, ui, packdir, version=0):
self._checkversion(version)
opener = vfsmod.vfs(packdir)
opener.createmode = 0o444
self.opener = opener
self.entries = {}
shallowutil.mkstickygroupdir(ui, packdir)
self.packfp, self.packpath = opener.mkstemp(
suffix=self.PACKSUFFIX + '-tmp')
self.idxfp, self.idxpath = opener.mkstemp(
suffix=self.INDEXSUFFIX + '-tmp')
self.packfp = os.fdopen(self.packfp, 'w+')
self.idxfp = os.fdopen(self.idxfp, 'w+')
self.sha = hashlib.sha1()
self._closed = False
# The opener provides no way of doing permission fixup on files created
# via mkstemp, so we must fix it ourselves. We can probably fix this
# upstream in vfs.mkstemp so we don't need to use the private method.
opener._fixfilemode(opener.join(self.packpath))
opener._fixfilemode(opener.join(self.idxpath))
# Write header
# TODO: make it extensible (ex: allow specifying compression algorithm,
# a flexible key/value header, delta algorithm, fanout size, etc)
versionbuf = struct.pack('!B', self.VERSION) # unsigned 1 byte int
self.writeraw(versionbuf)
def __init__(self, testcase, dir, pidoffset=0):
self._testcase = testcase
self._acquirecalled = False
self._releasecalled = False
self._postreleasecalled = False
self.vfs = vfsmod.vfs(dir, audit=False)
self._pidoffset = pidoffset
def openlfdirstate(ui, repo, create=True):
'''
Return a dirstate object that tracks largefiles: i.e. its root is
the repo root, but it is saved in .hg/largefiles/dirstate.
'''
vfs = repo.vfs
lfstoredir = longname
opener = vfsmod.vfs(vfs.join(lfstoredir))
lfdirstate = largefilesdirstate(opener, ui, repo.root,
repo.dirstate._validate,
lambda: sparse.matcher(repo))
# If the largefiles dirstate does not exist, populate and create
# it. This ensures that we create it on the first meaningful
# largefiles operation in a new clone.
if create and not vfs.exists(vfs.join(lfstoredir, 'dirstate')):
matcher = getstandinmatcher(repo)
standins = repo.dirstate.walk(matcher, subrepos=[], unknown=False,
ignored=False)
if len(standins) > 0:
vfs.makedirs(lfstoredir)
for standin in standins:
lfile = splitstandin(standin)
lfdirstate.normallookup(lfile)
return lfdirstate
def deletesharedstore(path=None):
storepath = path or destvfs.read(b".hg/sharedpath").strip()
if storepath.endswith(b".hg"):
storepath = os.path.dirname(storepath)
storevfs = vfs.vfs(storepath, audit=False)
storevfs.rmtree(forcibly=True)
def __init__(self, ui, packdir, version=2):
self._checkversion(version)
# TODO(augie): make this configurable
self._compressor = b'GZ'
opener = vfsmod.vfs(packdir)
opener.createmode = 0o444
self.opener = opener
self.entries = {}
shallowutil.mkstickygroupdir(ui, packdir)
self.packfp, self.packpath = opener.mkstemp(suffix=self.PACKSUFFIX +
b'-tmp')
self.idxfp, self.idxpath = opener.mkstemp(suffix=self.INDEXSUFFIX +
b'-tmp')
self.packfp = os.fdopen(self.packfp, r'wb+')
self.idxfp = os.fdopen(self.idxfp, r'wb+')
self.sha = hashlib.sha1()
self._closed = False
# The opener provides no way of doing permission fixup on files created
# via mkstemp, so we must fix it ourselves. We can probably fix this
# upstream in vfs.mkstemp so we don't need to use the private method.
opener._fixfilemode(opener.join(self.packpath))
opener._fixfilemode(opener.join(self.idxpath))
# Write header
# TODO: make it extensible (ex: allow specifying compression algorithm,
# a flexible key/value header, delta algorithm, fanout size, etc)
versionbuf = struct.pack(b'!B', self.VERSION) # unsigned 1 byte int
self.writeraw(versionbuf)
def _gitvfs(repo):
"""return a vfs suitable to read git related data"""
# Mercurial >= 3.3: repo.shared()
if repo.sharedpath != repo.path:
return vfsmod.vfs(repo.sharedpath)
else:
return repo.vfs
def repacklockvfs(repo):
if util.safehasattr(repo, 'name'):
# Lock in the shared cache so repacks across multiple copies of the same
# repo are coordinated.
sharedcachepath = shallowutil.getcachepackpath(
repo, constants.FILEPACK_CATEGORY)
return vfs.vfs(sharedcachepath)
else:
return repo.svfs
def repacklockvfs(repo):
if util.safehasattr(repo, 'name'):
# Lock in the shared cache so repacks across multiple copies of the same
# repo are coordinated.
sharedcachepath = shallowutil.getcachepackpath(
repo,
constants.FILEPACK_CATEGORY)
return vfs.vfs(sharedcachepath)
else:
return repo.svfs
def __init__(self, ui, repo, opts):
self.ui = ui
self.path = repo.vfs.join('transplant')
self.opener = vfsmod.vfs(self.path)
self.transplants = transplants(self.path, 'transplants',
opener=self.opener)
def getcommiteditor():
editform = cmdutil.mergeeditform(repo[None], 'transplant')
return cmdutil.getcommiteditor(editform=editform, **opts)
self.getcommiteditor = getcommiteditor
def __init__(self, path=None, transplantfile=None, opener=None):
self.path = path
self.transplantfile = transplantfile
self.opener = opener
if not opener:
self.opener = vfsmod.vfs(self.path)
self.transplants = {}
self.dirty = False
self.read()
def testflock(self):
testtmp = os.environ["TESTTMP"]
opener = vfs.vfs(testtmp)
name = 'testlock'
with extutil.flock(opener.join(name), 'testing a lock',
timeout=0):
otherlock = self.otherprocesslock(opener, name)
self.assertEquals(otherlock, locktimeout,
"other process should not have taken the lock")
otherlock = self.otherprocesslock(opener, name)
self.assertEquals(otherlock, locksuccess,
"other process should have taken the lock")
def cleanupoldbackups(repo):
vfs = vfsmod.vfs(repo.vfs.join(backupdir))
maxbackups = repo.ui.configint('shelve', 'maxbackups')
hgfiles = [f for f in vfs.listdir() if f.endswith('.' + patchextension)]
hgfiles = sorted([(vfs.stat(f)[stat.ST_MTIME], f) for f in hgfiles])
if maxbackups > 0 and maxbackups < len(hgfiles):
bordermtime = hgfiles[-maxbackups][0]
else:
bordermtime = None
for mtime, f in hgfiles[:len(hgfiles) - maxbackups]:
if mtime == bordermtime:
# keep it, because timestamp can't decide exact order of backups
continue
base = f[:-(1 + len(patchextension))]
for ext in shelvefileextensions:
vfs.tryunlink(base + '.' + ext)
def debugfastpartialmatchstat(ui, repo):
if not repo.svfs.exists(_partialindexdir):
ui.warn(_('partial index is not built\n'))
return 1
generationnum = _readgenerationnum(ui, repo.svfs)
ui.write(_('generation number: %d\n') % generationnum)
if _needsrebuilding(repo):
ui.write(_('index will be rebuilt on the next pull\n'))
indexvfs = vfsmod.vfs(repo.svfs.join(_partialindexdir))
for indexfile in sorted(_iterindexfile(indexvfs)):
size = indexvfs.stat(indexfile).st_size - _header.headersize
entriescount = size / _entrysize
with indexvfs(indexfile) as fileobj:
header = _header.read(fileobj)
ui.write(_('file: %s, entries: %d, out of them %d sorted\n') %
(indexfile, entriescount, header.sortedcount))
def cleanupoldbackups(repo):
vfs = vfsmod.vfs(repo.vfs.join(backupdir))
maxbackups = repo.ui.configint('obsshelve', 'maxbackups')
hgfiles = [f for f in vfs.listdir()
if f.endswith('.' + patchextension)]
hgfiles = sorted([(vfs.stat(f).st_mtime, f) for f in hgfiles])
if 0 < maxbackups and maxbackups < len(hgfiles):
bordermtime = hgfiles[-maxbackups][0]
else:
bordermtime = None
for mtime, f in hgfiles[:len(hgfiles) - maxbackups]:
if mtime == bordermtime:
# keep it, because timestamp can't decide exact order of backups
continue
base = f[:-(1 + len(patchextension))]
for ext in shelvefileextensions:
vfs.tryunlink(base + '.' + ext)
def share(orig, ui, source, *args, **kwargs):
"""Wraps hg.share to mark the firefoxtrees file as shared.
The .hg/shared file lists things that are shared. We add firefoxtrees
to it if we are a Firefox repo.
"""
res = orig(ui, source, *args, **kwargs)
# TODO Mercurial 3.7 introduces a standalone function that receives the
# proper arguments so we can avoid this boilerplate.
if isinstance(source, str):
origsource = ui.expandpath(source)
source, branches = hg.parseurl(origsource)
srcrepo = hg.repository(ui, source)
else:
srcrepo = source.local()
if not isfirefoxrepo(srcrepo):
return res
if args:
dest = args[0]
elif 'dest' in kwargs:
dest = kwargs['dest']
else:
dest = None
if not dest:
dest = hg.defaultdest(source)
else:
dest = ui.expandpath(dest)
try:
from mercurial.vfs import vfs
except ImportError:
vfs = scmutil.vfs
destwvfs = vfs(dest, realpath=True)
r = hg.repository(ui, destwvfs.base)
with r.wlock():
with r.vfs('shared', 'ab') as fh:
fh.write('firefoxtrees\n')
return res
def flock(lockpath, description, timeout=-1):
"""A flock based lock object. Currently it is always non-blocking.
Note that since it is flock based, you can accidentally take it multiple
times within one process and the first one to be released will release all
of them. So the caller needs to be careful to not create more than one
instance per lock.
"""
# best effort lightweight lock
try:
import fcntl
fcntl.flock
except ImportError:
# fallback to Mercurial lock
vfs = vfsmod.vfs(os.path.dirname(lockpath))
with lockmod.lock(vfs, os.path.basename(lockpath), timeout=timeout):
yield
return
# make sure lock file exists
util.makedirs(os.path.dirname(lockpath))
with open(lockpath, 'a'):
pass
lockfd = os.open(lockpath, os.O_RDONLY, 0o664)
start = time.time()
while True:
try:
fcntl.flock(lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except IOError as ex:
if ex.errno == errno.EAGAIN:
if timeout != -1 and time.time() - start > timeout:
raise error.LockHeld(errno.EAGAIN, lockpath, description,
'')
else:
time.sleep(0.05)
continue
raise
try:
yield
finally:
fcntl.flock(lockfd, fcntl.LOCK_UN)
os.close(lockfd)
def share(orig, ui, source, *args, **kwargs):
"""Wraps hg.share to mark the firefoxtrees file as shared.
The .hg/shared file lists things that are shared. We add firefoxtrees
to it if we are a Firefox repo.
"""
res = orig(ui, source, *args, **kwargs)
if not util.safehasattr(source, 'local'):
origsource = ui.expandpath(source)
source, branches = hg.parseurl(origsource)
srcrepo = hg.repository(ui, source)
else:
srcrepo = source.local()
if not isfirefoxrepo(srcrepo):
return res
if args:
dest = args[0]
elif 'dest' in kwargs:
dest = kwargs['dest']
else:
dest = None
if not dest:
dest = hg.defaultdest(source)
else:
dest = ui.expandpath(dest)
try:
from mercurial.vfs import vfs
except ImportError:
vfs = scmutil.vfs
destwvfs = vfs(dest, realpath=True)
r = hg.repository(ui, destwvfs.base)
with r.wlock():
with r.vfs(b'shared', b'ab') as fh:
fh.write(b'firefoxtrees\n')
return res
def openlfdirstate(ui, repo, create=True):
"""
Return a dirstate object that tracks largefiles: i.e. its root is
the repo root, but it is saved in .hg/largefiles/dirstate.
"""
vfs = repo.vfs
lfstoredir = longname
opener = vfsmod.vfs(vfs.join(lfstoredir))
use_dirstate_v2 = requirements.DIRSTATE_V2_REQUIREMENT in repo.requirements
lfdirstate = largefilesdirstate(
opener,
ui,
repo.root,
repo.dirstate._validate,
lambda: sparse.matcher(repo),
repo.nodeconstants,
use_dirstate_v2,
)
# If the largefiles dirstate does not exist, populate and create
# it. This ensures that we create it on the first meaningful
# largefiles operation in a new clone.
if create and not vfs.exists(vfs.join(lfstoredir, b'dirstate')):
matcher = getstandinmatcher(repo)
standins = repo.dirstate.walk(
matcher, subrepos=[], unknown=False, ignored=False
)
if len(standins) > 0:
vfs.makedirs(lfstoredir)
with lfdirstate.parentchange():
for standin in standins:
lfile = splitstandin(standin)
lfdirstate.update_file(
lfile, p1_tracked=True, wc_tracked=True, possibly_dirty=True
)
return lfdirstate
def debugcheckfastpartialindex(ui, repo):
'''Command to check that partial index is consistent
It checks that revision numbers are correct and checks that partial index
has all the nodes from the repo.
'''
if not repo.svfs.exists(_partialindexdir):
ui.warn(_('partial index is not built\n'))
return 1
indexvfs = vfsmod.vfs(repo.svfs.join(_partialindexdir))
foundnodes = set()
# Use unfiltered repo because index may have entries that point to hidden
# commits
ret = 0
repo = repo.unfiltered()
for indexfile in _iterindexfile(indexvfs):
try:
for node, actualrev in _parseindexfile(indexvfs, indexfile):
expectedrev = repo.changelog.rev(node)
foundnodes.add(node)
if expectedrev != actualrev:
ret = 1
ui.warn(_('corrupted index: rev number for %s ' +
'should be %d but found %d\n') %
(hex(node), expectedrev, actualrev))
except ValueError as e:
ret = 1
ui.warn(_('%s file is corrupted: %s\n') % (indexfile, e))
for rev in repo:
node = repo[rev].node()
if node not in foundnodes:
ret = 1
ui.warn(_('%s node not found in partialindex\n') % hex(node))
return ret
def _docheckout(
ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
optimes,
behaviors,
networkattemptlimit,
networkattempts=None,
sparse_profile=None,
noupdate=False,
):
if not networkattempts:
networkattempts = [1]
def callself():
return _docheckout(
ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
optimes,
behaviors,
networkattemptlimit,
networkattempts=networkattempts,
sparse_profile=sparse_profile,
noupdate=noupdate,
)
@contextlib.contextmanager
def timeit(op, behavior):
behaviors.add(behavior)
errored = False
try:
start = time.time()
yield
except Exception:
errored = True
raise
finally:
elapsed = time.time() - start
if errored:
op += "_errored"
optimes.append((op, elapsed))
ui.write(b"ensuring %s@%s is available at %s\n" % (url, revision or branch, dest))
# We assume that we're the only process on the machine touching the
# repository paths that we were told to use. This means our recovery
# scenario when things aren't "right" is to just nuke things and start
# from scratch. This is easier to implement than verifying the state
# of the data and attempting recovery. And in some scenarios (such as
# potential repo corruption), it is probably faster, since verifying
# repos can take a while.
destvfs = vfs.vfs(dest, audit=False, realpath=True)
def deletesharedstore(path=None):
storepath = path or destvfs.read(b".hg/sharedpath").strip()
if storepath.endswith(b".hg"):
storepath = os.path.dirname(storepath)
storevfs = vfs.vfs(storepath, audit=False)
storevfs.rmtree(forcibly=True)
if destvfs.exists() and not destvfs.exists(b".hg"):
raise error.Abort(b"destination exists but no .hg directory")
# Refuse to enable sparse checkouts on existing checkouts. The reasoning
# here is that another consumer of this repo may not be sparse aware. If we
# enabled sparse, we would lock them out.
if destvfs.exists() and sparse_profile and not destvfs.exists(b".hg/sparse"):
raise error.Abort(
b"cannot enable sparse profile on existing " b"non-sparse checkout",
hint=b"use a separate working directory to use sparse",
)
# And the other direction for symmetry.
if not sparse_profile and destvfs.exists(b".hg/sparse"):
raise error.Abort(
b"cannot use non-sparse checkout on existing sparse " b"checkout",
hint=b"use a separate working directory to use sparse",
)
# Require checkouts to be tied to shared storage because efficiency.
if destvfs.exists(b".hg") and not destvfs.exists(b".hg/sharedpath"):
ui.warn(b"(destination is not shared; deleting)\n")
with timeit("remove_unshared_dest", "remove-wdir"):
destvfs.rmtree(forcibly=True)
# Verify the shared path exists and is using modern pooled storage.
if destvfs.exists(b".hg/sharedpath"):
storepath = destvfs.read(b".hg/sharedpath").strip()
ui.write(b"(existing repository shared store: %s)\n" % storepath)
if not os.path.exists(storepath):
ui.warn(b"(shared store does not exist; deleting destination)\n")
with timeit("removed_missing_shared_store", "remove-wdir"):
destvfs.rmtree(forcibly=True)
elif not re.search(b"[a-f0-9]{40}/\.hg$", storepath.replace(b"\\", b"/")):
ui.warn(
b"(shared store does not belong to pooled storage; "
b"deleting destination to improve efficiency)\n"
)
with timeit("remove_unpooled_store", "remove-wdir"):
destvfs.rmtree(forcibly=True)
if destvfs.isfileorlink(b".hg/wlock"):
ui.warn(
b"(dest has an active working directory lock; assuming it is "
b"left over from a previous process and that the destination "
b"is corrupt; deleting it just to be sure)\n"
)
with timeit("remove_locked_wdir", "remove-wdir"):
destvfs.rmtree(forcibly=True)
def handlerepoerror(e):
if pycompat.bytestr(e) == _(b"abandoned transaction found"):
ui.warn(b"(abandoned transaction found; trying to recover)\n")
repo = hg.repository(ui, dest)
if not repo.recover():
ui.warn(b"(could not recover repo state; " b"deleting shared store)\n")
with timeit("remove_unrecovered_shared_store", "remove-store"):
deletesharedstore()
ui.warn(b"(attempting checkout from beginning)\n")
return callself()
raise
# At this point we either have an existing working directory using
# shared, pooled storage or we have nothing.
def handlenetworkfailure():
if networkattempts[0] >= networkattemptlimit:
raise error.Abort(
b"reached maximum number of network attempts; " b"giving up\n"
)
ui.warn(
b"(retrying after network failure on attempt %d of %d)\n"
% (networkattempts[0], networkattemptlimit)
)
# Do a backoff on retries to mitigate the thundering herd
# problem. This is an exponential backoff with a multipler
# plus random jitter thrown in for good measure.
# With the default settings, backoffs will be:
# 1) 2.5 - 6.5
# 2) 5.5 - 9.5
# 3) 11.5 - 15.5
backoff = (2 ** networkattempts[0] - 1) * 1.5
jittermin = ui.configint(b"robustcheckout", b"retryjittermin", 1000)
jittermax = ui.configint(b"robustcheckout", b"retryjittermax", 5000)
backoff += float(random.randint(jittermin, jittermax)) / 1000.0
ui.warn(b"(waiting %.2fs before retry)\n" % backoff)
time.sleep(backoff)
networkattempts[0] += 1
def handlepullerror(e):
"""Handle an exception raised during a pull.
Returns True if caller should call ``callself()`` to retry.
"""
if isinstance(e, error.Abort):
if e.args[0] == _(b"repository is unrelated"):
ui.warn(b"(repository is unrelated; deleting)\n")
destvfs.rmtree(forcibly=True)
return True
elif e.args[0].startswith(_(b"stream ended unexpectedly")):
ui.warn(b"%s\n" % e.args[0])
# Will raise if failure limit reached.
handlenetworkfailure()
return True
# TODO test this branch
elif isinstance(e, error.ResponseError):
if e.args[0].startswith(_(b"unexpected response from remote server:")):
ui.warn(b"(unexpected response from remote server; retrying)\n")
destvfs.rmtree(forcibly=True)
# Will raise if failure limit reached.
handlenetworkfailure()
return True
elif isinstance(e, ssl.SSLError):
# Assume all SSL errors are due to the network, as Mercurial
# should convert non-transport errors like cert validation failures
# to error.Abort.
ui.warn(b"ssl error: %s\n" % e)
handlenetworkfailure()
return True
elif isinstance(e, urllibcompat.urlerr.urlerror):
if isinstance(e.reason, socket.error):
ui.warn(b"socket error: %s\n" % pycompat.bytestr(e.reason))
handlenetworkfailure()
return True
else:
ui.warn(
b"unhandled URLError; reason type: %s; value: %s\n"
% (e.reason.__class__.__name__, e.reason)
)
else:
ui.warn(
b"unhandled exception during network operation; type: %s; "
b"value: %s\n" % (e.__class__.__name__, e)
)
return False
# Perform sanity checking of store. We may or may not know the path to the
# local store. It depends if we have an existing destvfs pointing to a
# share. To ensure we always find a local store, perform the same logic
# that Mercurial's pooled storage does to resolve the local store path.
cloneurl = upstream or url
try:
clonepeer = hg.peer(ui, {}, cloneurl)
rootnode = peerlookup(clonepeer, b"0")
except error.RepoLookupError:
raise error.Abort(b"unable to resolve root revision from clone " b"source")
except (error.Abort, ssl.SSLError, urllibcompat.urlerr.urlerror) as e:
if handlepullerror(e):
return callself()
raise
if rootnode == nullid:
raise error.Abort(b"source repo appears to be empty")
storepath = os.path.join(sharebase, hex(rootnode))
storevfs = vfs.vfs(storepath, audit=False)
if storevfs.isfileorlink(b".hg/store/lock"):
ui.warn(
b"(shared store has an active lock; assuming it is left "
b"over from a previous process and that the store is "
b"corrupt; deleting store and destination just to be "
b"sure)\n"
)
if destvfs.exists():
with timeit("remove_dest_active_lock", "remove-wdir"):
destvfs.rmtree(forcibly=True)
with timeit("remove_shared_store_active_lock", "remove-store"):
storevfs.rmtree(forcibly=True)
if storevfs.exists() and not storevfs.exists(b".hg/requires"):
ui.warn(
b"(shared store missing requires file; this is a really "
b"odd failure; deleting store and destination)\n"
)
if destvfs.exists():
with timeit("remove_dest_no_requires", "remove-wdir"):
destvfs.rmtree(forcibly=True)
with timeit("remove_shared_store_no_requires", "remove-store"):
storevfs.rmtree(forcibly=True)
if storevfs.exists(b".hg/requires"):
requires = set(storevfs.read(b".hg/requires").splitlines())
# FUTURE when we require generaldelta, this is where we can check
# for that.
required = {b"dotencode", b"fncache"}
missing = required - requires
if missing:
ui.warn(
b"(shared store missing requirements: %s; deleting "
b"store and destination to ensure optimal behavior)\n"
% b", ".join(sorted(missing))
)
if destvfs.exists():
with timeit("remove_dest_missing_requires", "remove-wdir"):
destvfs.rmtree(forcibly=True)
with timeit("remove_shared_store_missing_requires", "remove-store"):
storevfs.rmtree(forcibly=True)
created = False
if not destvfs.exists():
# Ensure parent directories of destination exist.
# Mercurial 3.8 removed ensuredirs and made makedirs race safe.
if util.safehasattr(util, "ensuredirs"):
makedirs = util.ensuredirs
else:
makedirs = util.makedirs
makedirs(os.path.dirname(destvfs.base), notindexed=True)
makedirs(sharebase, notindexed=True)
if upstream:
ui.write(b"(cloning from upstream repo %s)\n" % upstream)
if not storevfs.exists():
behaviors.add(b"create-store")
try:
with timeit("clone", "clone"):
shareopts = {b"pool": sharebase, b"mode": b"identity"}
res = hg.clone(
ui,
{},
clonepeer,
dest=dest,
update=False,
shareopts=shareopts,
stream=True,
)
except (error.Abort, ssl.SSLError, urllibcompat.urlerr.urlerror) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn(b"(repo corruption: %s; deleting shared store)\n" % e)
with timeit("remove_shared_store_revlogerror", "remote-store"):
deletesharedstore()
return callself()
# TODO retry here.
if res is None:
raise error.Abort(b"clone failed")
# Verify it is using shared pool storage.
if not destvfs.exists(b".hg/sharedpath"):
raise error.Abort(b"clone did not create a shared repo")
created = True
# The destination .hg directory should exist. Now make sure we have the
# wanted revision.
repo = hg.repository(ui, dest)
# We only pull if we are using symbolic names or the requested revision
# doesn't exist.
havewantedrev = False
if revision:
try:
ctx = scmutil.revsingle(repo, revision)
except error.RepoLookupError:
ctx = None
if ctx:
if not ctx.hex().startswith(revision):
raise error.Abort(
b"--revision argument is ambiguous",
hint=b"must be the first 12+ characters of a " b"SHA-1 fragment",
)
checkoutrevision = ctx.hex()
havewantedrev = True
if not havewantedrev:
ui.write(b"(pulling to obtain %s)\n" % (revision or branch,))
remote = None
try:
remote = hg.peer(repo, {}, url)
pullrevs = [peerlookup(remote, revision or branch)]
checkoutrevision = hex(pullrevs[0])
if branch:
ui.warn(
b"(remote resolved %s to %s; "
b"result is not deterministic)\n" % (branch, checkoutrevision)
)
if checkoutrevision in repo:
ui.warn(b"(revision already present locally; not pulling)\n")
else:
with timeit("pull", "pull"):
pullop = exchange.pull(repo, remote, heads=pullrevs)
if not pullop.rheads:
raise error.Abort(b"unable to pull requested revision")
except (error.Abort, ssl.SSLError, urllibcompat.urlerr.urlerror) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn(b"(repo corruption: %s; deleting shared store)\n" % e)
deletesharedstore()
return callself()
finally:
if remote:
remote.close()
# Now we should have the wanted revision in the store. Perform
# working directory manipulation.
# Avoid any working directory manipulations if `-U`/`--noupdate` was passed
if noupdate:
ui.write(b"(skipping update since `-U` was passed)\n")
return None
# Purge if requested. We purge before update because this way we're
# guaranteed to not have conflicts on `hg update`.
if purge and not created:
ui.write(b"(purging working directory)\n")
purgeext = extensions.find(b"purge")
# Mercurial 4.3 doesn't purge files outside the sparse checkout.
# See https://bz.mercurial-scm.org/show_bug.cgi?id=5626. Force
# purging by monkeypatching the sparse matcher.
try:
old_sparse_fn = getattr(repo.dirstate, "_sparsematchfn", None)
if old_sparse_fn is not None:
# TRACKING hg50
# Arguments passed to `matchmod.always` were unused and have been removed
if util.versiontuple(n=2) >= (5, 0):
repo.dirstate._sparsematchfn = lambda: matchmod.always()
else:
repo.dirstate._sparsematchfn = lambda: matchmod.always(
repo.root, ""
)
with timeit("purge", "purge"):
if purgeext.purge(
ui,
repo,
all=True,
abort_on_err=True,
# The function expects all arguments to be
# defined.
**{"print": None, "print0": None, "dirs": None, "files": None}
):
raise error.Abort(b"error purging")
finally:
if old_sparse_fn is not None:
repo.dirstate._sparsematchfn = old_sparse_fn
# Update the working directory.
if repo[b"."].node() == nullid:
behaviors.add("empty-wdir")
else:
behaviors.add("populated-wdir")
if sparse_profile:
sparsemod = getsparse()
# By default, Mercurial will ignore unknown sparse profiles. This could
# lead to a full checkout. Be more strict.
try:
repo.filectx(sparse_profile, changeid=checkoutrevision).data()
except error.ManifestLookupError:
raise error.Abort(
b"sparse profile %s does not exist at revision "
b"%s" % (sparse_profile, checkoutrevision)
)
# TRACKING hg48 - parseconfig takes `action` param
if util.versiontuple(n=2) >= (4, 8):
old_config = sparsemod.parseconfig(
repo.ui, repo.vfs.tryread(b"sparse"), b"sparse"
)
else:
old_config = sparsemod.parseconfig(repo.ui, repo.vfs.tryread(b"sparse"))
old_includes, old_excludes, old_profiles = old_config
if old_profiles == {sparse_profile} and not old_includes and not old_excludes:
ui.write(
b"(sparse profile %s already set; no need to update "
b"sparse config)\n" % sparse_profile
)
else:
if old_includes or old_excludes or old_profiles:
ui.write(
b"(replacing existing sparse config with profile "
b"%s)\n" % sparse_profile
)
else:
ui.write(b"(setting sparse config to profile %s)\n" % sparse_profile)
# If doing an incremental update, this will perform two updates:
# one to change the sparse profile and another to update to the new
# revision. This is not desired. But there's not a good API in
# Mercurial to do this as one operation.
with repo.wlock(), timeit("sparse_update_config", "sparse-update-config"):
# pylint --py3k: W1636
fcounts = list(
map(
len,
sparsemod._updateconfigandrefreshwdir(
repo, [], [], [sparse_profile], force=True
),
)
)
repo.ui.status(
b"%d files added, %d files dropped, "
b"%d files conflicting\n" % tuple(fcounts)
)
ui.write(b"(sparse refresh complete)\n")
op = "update_sparse" if sparse_profile else "update"
behavior = "update-sparse" if sparse_profile else "update"
with timeit(op, behavior):
if commands.update(ui, repo, rev=checkoutrevision, clean=True):
raise error.Abort(b"error updating")
ui.write(b"updated to %s\n" % checkoutrevision)
return None
def profiletreepack(repo, store, rev1, rev2, opts):
def exectest(name, count, prep, func):
elapsed = 0
elapsedprep = 0
for i in xrange(0, count):
args = []
if prep:
startprep = time.time()
args = prep()
elapsedprep += time.time() - startprep
import gc
gc.disable()
start = time.time()
func(*args)
elapsed += time.time() - start
gc.enable()
gc.collect()
repo.ui.progress(name, i, total=count)
repo.ui.progress(name, None)
total = elapsed + elapsedprep
repo.ui.status(("%0.2f (%0.2f)" % (elapsedprep, elapsedprep / (count *
1.0))).ljust(15))
repo.ui.status(("%0.2f (%0.2f)" % (elapsed, elapsed / (count *
1.0))).ljust(15))
repo.ui.status(("%0.2f (%0.2f)" % (total, total / (count *
1.0))).ljust(15))
ctx1 = list(repo.set(rev1))[0]
ctx2 = list(repo.set(rev2))[0]
cachemanager.cachemanifestfillandtrim(
repo.ui, repo, ['%s + %s' % (ctx1.rev(), ctx2.rev())])
def treeconstructor(mfnode):
return read(store, '', mfnode)
def flatconstructor(mfnode):
repo.manifest.clearcaches()
return repo.manifest.read(mfnode)._flatmanifest()
cacheopener = vfsmod.vfs(repo.vfs.join('cache', 'fastmanifest_test'))
if not os.path.exists(cacheopener.base):
os.mkdir(cacheopener.base)
manager = fastmanifestcache.getinstance(cacheopener, repo.ui)
def fastconstructor(mfnode):
repo.manifest.clearcaches()
manager.inmemorycache.clear()
return repo.manifest.read(mfnode)._cachedmanifest()
class FakeStore(object):
def __init__(self):
self._cache = {}
def get(self, filename, node):
try:
return self._cache[(filename, node)]
except KeyError:
result = store.get(filename, node)
self._cache[(filename, node)] = result
return result
def ctreeconstructor(mfnode):
# Enable the fake store to remove blob lookup time
treemf = cstore.treemanifest(store, mfnode)
return treemf
# Test bodies
def prepone(new):
return [new(ctx1.manifestnode())]
def preptwo(new):
m1 = new(ctx1.manifestnode())
m2 = new(ctx2.manifestnode())
return m1, m2
def diff(m1, m2):
diff = m1.diff(m2)
if len(diff) < 10000:
raise error.Abort("diff only found %s items" % len(diff))
def find(m1):
findresult = m1.find('fbcode/hphp/test/run')
if findresult == (None, None):
raise error.Abort("None find result")
def fulliter(m1):
count = 0
for x in m1:
count += 1
if count < 900000:
raise error.Abort("fulliter only found %s files" % count)
kindconstructor = {
'tree': treeconstructor,
'flat': flatconstructor,
'fast': fastconstructor,
'ctree': ctreeconstructor,
}
iterations = int(opts.get('iterations'))
testopts = {
'diff': (preptwo, diff, iterations),
'find': (prepone, find, iterations),
'fulliter': (prepone, fulliter, max(1, iterations / 2)),
}
tests = opts.get('test', 'all').split(',')
if tests[0] in ['all', '']:
tests = testopts.keys()
kinds = opts.get('kind', 'all').split(',')
if kinds[0] in ['all', '']:
kinds = kindconstructor.keys()
# Prime any caches
for kind in kinds:
preptwo(kindconstructor[kind])
for test in tests:
prepfunc, func, iterations = testopts[test]
teststr = ('%s (%s)' % (test, iterations)).ljust(14)
repo.ui.status(("\n%sPrep Run Total\n") %
(teststr))
for kind in kinds:
repo.ui.status(("%s" % (kind)).ljust(14))
def prep():
return prepfunc(kindconstructor[kind])
exectest(test, iterations, prep, func)
repo.ui.status("\n")
repo.ui.status("\n")
def __init__(self, baseui, path, create=False, intents=None):
# type: (uimod.ui, str, bool, Any) -> None
self.requirements = set()
self.filtername = None
self._phasedefaults = []
self._bookmarks = {}
self.obsstore = False
self._revbranchcache = None
# generic mapping between names and nodes
self.names = gitnamespaces()
# wvfs: rooted at the repository root, used to access the working copy
self.wvfs = vfsmod.vfs(path, expandpath=True, realpath=True)
# vfs: rooted at .hg, used to access repo files outside of .hg/store
self.vfs = None
# svfs: usually rooted at .hg/store, used to access repository history
# If this is a shared repository, this vfs may point to another
# repository's .hg/store directory.
self.svfs = None
self.root = self.wvfs.base
self.path = self.wvfs.join(".git")
self.origroot = path
# This is only used by context.workingctx.match in order to
# detect files in subrepos.
self.auditor = pathutil.pathauditor(self.root,
callback=self._checknested)
# This is only used by context.basectx.match in order to detect
# files in subrepos.
self.nofsauditor = pathutil.pathauditor(self.root,
callback=self._checknested,
realfs=False,
cached=True)
self.baseui = baseui
self.ui = baseui.copy()
self.ui.copy = baseui.copy # prevent copying repo configuration
self.vfs = vfsmod.vfs(self.path, cacheaudited=True)
if (self.ui.configbool('devel', 'all-warnings')
or self.ui.configbool('devel', 'check-locks')):
self.vfs.audit = self._getvfsward(self.vfs.audit)
try:
self.ui.readconfig(self.vfs.join("hgrc"), self.root)
# self._loadextensions()
except IOError:
pass
color.setup(self.ui)
if not self.vfs.isdir():
if create:
pygit2.init_repository(self.path, False)
else:
raise error.RepoError(_("repository %s not found") % path)
elif create:
raise error.RepoError(_("repository %s already exists") % path)
self._repo = pygit2.Repository(self.path)
# FIXME: move to propertycache
self.dirstate = gitdirstate(self, self.ui, self.root)
def main():
ui = uimod.ui()
# Needed so we can open a local repo with obsstore without a warning.
ui.setconfig(b'experimental', b'evolution.createmarkers', True)
checkzobject(badpeer())
ziverify.verifyClass(repository.ipeerbase, httppeer.httppeer)
checkzobject(httppeer.httppeer(None, None, None, dummyopener(), None,
None))
ziverify.verifyClass(repository.ipeerv2, httppeer.httpv2peer)
checkzobject(httppeer.httpv2peer(None, b'', b'', None, None, None))
ziverify.verifyClass(repository.ipeerbase, localrepo.localpeer)
checkzobject(localrepo.localpeer(dummyrepo()))
ziverify.verifyClass(repository.ipeercommandexecutor,
localrepo.localcommandexecutor)
checkzobject(localrepo.localcommandexecutor(None))
ziverify.verifyClass(repository.ipeercommandexecutor,
wireprotov1peer.peerexecutor)
checkzobject(wireprotov1peer.peerexecutor(None))
ziverify.verifyClass(repository.ipeerbase, sshpeer.sshv1peer)
checkzobject(
sshpeer.sshv1peer(ui, b'ssh://localhost/foo', b'', dummypipe(),
dummypipe(), None, None))
ziverify.verifyClass(repository.ipeerbase, sshpeer.sshv2peer)
checkzobject(
sshpeer.sshv2peer(ui, b'ssh://localhost/foo', b'', dummypipe(),
dummypipe(), None, None))
ziverify.verifyClass(repository.ipeerbase, bundlerepo.bundlepeer)
checkzobject(bundlerepo.bundlepeer(dummyrepo()))
ziverify.verifyClass(repository.ipeerbase, statichttprepo.statichttppeer)
checkzobject(statichttprepo.statichttppeer(dummyrepo()))
ziverify.verifyClass(repository.ipeerbase, unionrepo.unionpeer)
checkzobject(unionrepo.unionpeer(dummyrepo()))
ziverify.verifyClass(repository.ilocalrepositorymain,
localrepo.localrepository)
ziverify.verifyClass(repository.ilocalrepositoryfilestorage,
localrepo.revlogfilestorage)
repo = localrepo.makelocalrepository(ui, rootdir)
checkzobject(repo)
ziverify.verifyClass(wireprototypes.baseprotocolhandler,
wireprotoserver.sshv1protocolhandler)
ziverify.verifyClass(wireprototypes.baseprotocolhandler,
wireprotoserver.sshv2protocolhandler)
ziverify.verifyClass(wireprototypes.baseprotocolhandler,
wireprotoserver.httpv1protocolhandler)
ziverify.verifyClass(wireprototypes.baseprotocolhandler,
wireprotov2server.httpv2protocolhandler)
sshv1 = wireprotoserver.sshv1protocolhandler(None, None, None)
checkzobject(sshv1)
sshv2 = wireprotoserver.sshv2protocolhandler(None, None, None)
checkzobject(sshv2)
httpv1 = wireprotoserver.httpv1protocolhandler(None, None, None)
checkzobject(httpv1)
httpv2 = wireprotov2server.httpv2protocolhandler(None, None)
checkzobject(httpv2)
ziverify.verifyClass(repository.ifilestorage, filelog.filelog)
ziverify.verifyClass(repository.imanifestdict, manifest.manifestdict)
ziverify.verifyClass(repository.imanifestrevisionstored,
manifest.manifestctx)
ziverify.verifyClass(repository.imanifestrevisionwritable,
manifest.memmanifestctx)
ziverify.verifyClass(repository.imanifestrevisionstored,
manifest.treemanifestctx)
ziverify.verifyClass(repository.imanifestrevisionwritable,
manifest.memtreemanifestctx)
ziverify.verifyClass(repository.imanifestlog, manifest.manifestlog)
ziverify.verifyClass(repository.imanifeststorage, manifest.manifestrevlog)
vfs = vfsmod.vfs(b'.')
fl = filelog.filelog(vfs, b'dummy.i')
checkzobject(fl, allowextra=True)
# Conforms to imanifestlog.
ml = manifest.manifestlog(vfs, repo, manifest.manifestrevlog(repo.svfs))
checkzobject(ml)
checkzobject(repo.manifestlog)
# Conforms to imanifestrevision.
mctx = ml[repo[0].manifestnode()]
checkzobject(mctx)
# Conforms to imanifestrevisionwritable.
checkzobject(mctx.new())
checkzobject(mctx.copy())
# Conforms to imanifestdict.
checkzobject(mctx.read())
mrl = manifest.manifestrevlog(vfs)
checkzobject(mrl)
ziverify.verifyClass(repository.irevisiondelta, revlog.revlogrevisiondelta)
rd = revlog.revlogrevisiondelta(node=b'',
p1node=b'',
p2node=b'',
basenode=b'',
linknode=b'',
flags=b'',
baserevisionsize=None,
revision=b'',
delta=None)
checkzobject(rd)
ziverify.verifyClass(repository.iverifyproblem, revlog.revlogproblem)
checkzobject(revlog.revlogproblem())
def __init__(self, repo):
self._repo = repo
# Mercurial 4.2 introduced the vfs module and deprecated scmutil.vfs.
try:
from mercurial.vfs import vfs
except ImportError:
vfs = scmutil.vfs
self._vfs = vfs(repo.vfs.join('reviewboard'), audit=False)
# Maps review identifiers to identifierrecord instances.
self._identifiers = {}
# Maps parent review id to identifierrecord instances. Shares the same
# object instances as _identifiers.
self._prids = {}
# Maps nodes to noderecord instances.
self._nodes = {}
self.baseurl = None
self.remoteurl = None
try:
for line in repo.vfs('reviews'):
line = line.strip()
if not line:
continue
fields = line.split(' ', 1)
if len(fields) != 2:
repo.ui.warn(_('malformed line in reviews file: %r\n') %
line)
continue
t, d = fields
# Identifier to parent review ID.
if t == 'p':
ident, rrid = d.split(' ', 1)
r = identifierrecord(parentrrid=rrid)
self._identifiers[ident] = r
self._prids[rrid] = r
# Node to review id.
elif t == 'c':
node, rid = d.split(' ', 1)
assert len(node) == 40
r = self._nodes.setdefault(bin(node), noderecord())
r.rrids.add(rid)
# Node to parent id.
elif t == 'pc':
node, pid = d.split(' ', 1)
assert len(node) == 40
self._nodes[bin(node)].parentrrids.add(pid)
elif t == 'u':
self.baseurl = d
elif t == 'r':
self.remoteurl = d
except IOError as inst:
if inst.errno != errno.ENOENT:
raise
def basic(repo):
print("* neither file exists")
# calls function
repo.cached
repo.invalidate()
print("* neither file still exists")
# uses cache
repo.cached
# create empty file
f = open('x', 'w')
f.close()
repo.invalidate()
print("* empty file x created")
# should recreate the object
repo.cached
f = open('x', 'w')
f.write('a')
f.close()
repo.invalidate()
print("* file x changed size")
# should recreate the object
repo.cached
repo.invalidate()
print("* nothing changed with either file")
# stats file again, reuses object
repo.cached
# atomic replace file, size doesn't change
# hopefully st_mtime doesn't change as well so this doesn't use the cache
# because of inode change
f = vfsmod.vfs('.')('x', 'w', atomictemp=True)
f.write('b')
f.close()
repo.invalidate()
print("* file x changed inode")
repo.cached
# create empty file y
f = open('y', 'w')
f.close()
repo.invalidate()
print("* empty file y created")
# should recreate the object
repo.cached
f = open('y', 'w')
f.write('A')
f.close()
repo.invalidate()
print("* file y changed size")
# should recreate the object
repo.cached
f = vfsmod.vfs('.')('y', 'w', atomictemp=True)
f.write('B')
f.close()
repo.invalidate()
print("* file y changed inode")
repo.cached
f = vfsmod.vfs('.')('x', 'w', atomictemp=True)
f.write('c')
f.close()
f = vfsmod.vfs('.')('y', 'w', atomictemp=True)
f.write('C')
f.close()
repo.invalidate()
print("* both files changed inode")
repo.cached
if sqlite3.sqlite_version_info < (3, 8, 3):
# WITH clause not supported
sqlitestore = None
except ImportError:
pass
try:
from mercurial import zstd
zstd.__version__
except ImportError:
zstd = None
STATE = {
'lastindex': 0,
'ui': uimod.ui(),
'vfs': vfsmod.vfs(b'.', realpath=True),
}
def makefilefn(self):
"""Factory for filelog instances."""
fl = filelog.filelog(STATE['vfs'], b'filelog-%d' % STATE['lastindex'])
STATE['lastindex'] += 1
return fl
def maketransaction(self):
vfsmap = {b'plain': STATE['vfs'], b'store': STATE['vfs']}
return transaction.transaction(STATE['ui'].warn, STATE['vfs'], vfsmap,
b'journal', b'undo')
def _getopener(path):
vfs = vfsmod.vfs(path)
vfs.createmode = 0o644
return vfs
# test revlog interaction about raw data (flagprocessor)
from __future__ import absolute_import, print_function
import sys
from mercurial import (
encoding,
node,
revlog,
transaction,
vfs,
)
# TESTTMP is optional. This makes it convenient to run without run-tests.py
tvfs = vfs.vfs(encoding.environ.get('TESTTMP', b'/tmp'))
# Enable generaldelta otherwise revlog won't use delta as expected by the test
tvfs.options = {'generaldelta': True, 'revlogv1': True}
# The test wants to control whether to use delta explicitly, based on
# "storedeltachains".
revlog.revlog._isgooddelta = lambda self, d, textlen: self.storedeltachains
def abort(msg):
print('abort: %s' % msg)
# Return 0 so run-tests.py could compare the output.
sys.exit()
def main():
ui = uimod.ui()
# Needed so we can open a local repo with obsstore without a warning.
ui.setconfig('experimental', 'evolution.createmarkers', True)
checkzobject(badpeer())
ziverify.verifyClass(repository.ipeerbase, httppeer.httppeer)
checkzobject(httppeer.httppeer(None, None, None, dummyopener(), None,
None))
ziverify.verifyClass(repository.ipeerconnection, httppeer.httpv2peer)
ziverify.verifyClass(repository.ipeercapabilities, httppeer.httpv2peer)
checkzobject(httppeer.httpv2peer(None, b'', b'', None, None, None))
ziverify.verifyClass(repository.ipeerbase, localrepo.localpeer)
checkzobject(localrepo.localpeer(dummyrepo()))
ziverify.verifyClass(repository.ipeercommandexecutor,
localrepo.localcommandexecutor)
checkzobject(localrepo.localcommandexecutor(None))
ziverify.verifyClass(repository.ipeercommandexecutor,
wireprotov1peer.peerexecutor)
checkzobject(wireprotov1peer.peerexecutor(None))
ziverify.verifyClass(repository.ipeerbase, sshpeer.sshv1peer)
checkzobject(
sshpeer.sshv1peer(ui, b'ssh://localhost/foo', b'', dummypipe(),
dummypipe(), None, None))
ziverify.verifyClass(repository.ipeerbase, sshpeer.sshv2peer)
checkzobject(
sshpeer.sshv2peer(ui, b'ssh://localhost/foo', b'', dummypipe(),
dummypipe(), None, None))
ziverify.verifyClass(repository.ipeerbase, bundlerepo.bundlepeer)
checkzobject(bundlerepo.bundlepeer(dummyrepo()))
ziverify.verifyClass(repository.ipeerbase, statichttprepo.statichttppeer)
checkzobject(statichttprepo.statichttppeer(dummyrepo()))
ziverify.verifyClass(repository.ipeerbase, unionrepo.unionpeer)
checkzobject(unionrepo.unionpeer(dummyrepo()))
ziverify.verifyClass(repository.completelocalrepository,
localrepo.localrepository)
repo = localrepo.localrepository(ui, rootdir)
checkzobject(repo)
ziverify.verifyClass(wireprototypes.baseprotocolhandler,
wireprotoserver.sshv1protocolhandler)
ziverify.verifyClass(wireprototypes.baseprotocolhandler,
wireprotoserver.sshv2protocolhandler)
ziverify.verifyClass(wireprototypes.baseprotocolhandler,
wireprotoserver.httpv1protocolhandler)
ziverify.verifyClass(wireprototypes.baseprotocolhandler,
wireprotov2server.httpv2protocolhandler)
sshv1 = wireprotoserver.sshv1protocolhandler(None, None, None)
checkzobject(sshv1)
sshv2 = wireprotoserver.sshv2protocolhandler(None, None, None)
checkzobject(sshv2)
httpv1 = wireprotoserver.httpv1protocolhandler(None, None, None)
checkzobject(httpv1)
httpv2 = wireprotov2server.httpv2protocolhandler(None, None)
checkzobject(httpv2)
ziverify.verifyClass(repository.ifilestorage, filelog.filelog)
vfs = vfsmod.vfs(b'.')
fl = filelog.filelog(vfs, b'dummy.i')
checkzobject(fl, allowextra=True)