def consumestreamclone(repo, fp):
"""Apply the contents from a streaming clone file.
This code is copied from Mercurial. Until Mercurial 3.5, this code was
a closure in wireproto.py and not consumeable by extensions.
"""
lock = repo.lock()
try:
repo.ui.status(_('streaming all changes\n'))
l = fp.readline()
try:
total_files, total_bytes = map(int, l.split(' ', 1))
except (ValueError, TypeError):
raise error.ResponseError(
_('unexpected response from remote server:'), l)
repo.ui.status(_('%d files to transfer, %s of data\n') %
(total_files, util.bytecount(total_bytes)))
handled_bytes = 0
repo.ui.progress(_('clone'), 0, total=total_bytes)
start = time.time()
tr = repo.transaction(_('clone'))
try:
for i in xrange(total_files):
# XXX doesn't support '\n' or '\r' in filenames
l = fp.readline()
try:
name, size = l.split('\0', 1)
size = int(size)
except (ValueError, TypeError):
raise error.ResponseError(
_('unexpected response from remote server:'), l)
if repo.ui.debugflag:
repo.ui.debug('adding %s (%s)\n' %
(name, util.bytecount(size)))
# for backwards compat, name was partially encoded
ofp = repo.svfs(store.decodedir(name), 'w')
for chunk in util.filechunkiter(fp, limit=size):
handled_bytes += len(chunk)
repo.ui.progress(_('clone'), handled_bytes,
total=total_bytes)
ofp.write(chunk)
ofp.close()
tr.close()
finally:
tr.release()
# Writing straight to files circumvented the inmemory caches
repo.invalidate()
elapsed = time.time() - start
if elapsed <= 0:
elapsed = 0.001
repo.ui.progress(_('clone'), None)
repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') %
(util.bytecount(total_bytes), elapsed,
util.bytecount(total_bytes / elapsed)))
finally:
lock.release()
def consumev1(repo, fp, filecount, bytecount):
"""Apply the contents from version 1 of a streaming clone file handle.
This takes the output from "streamout" and applies it to the specified
repository.
Like "streamout," the status line added by the wire protocol is not handled
by this function.
"""
lock = repo.lock()
try:
repo.ui.status(
_('%d files to transfer, %s of data\n') %
(filecount, util.bytecount(bytecount)))
handled_bytes = 0
repo.ui.progress(_('clone'), 0, total=bytecount)
start = time.time()
tr = repo.transaction(_('clone'))
try:
for i in xrange(filecount):
# XXX doesn't support '\n' or '\r' in filenames
l = fp.readline()
try:
name, size = l.split('\0', 1)
size = int(size)
except (ValueError, TypeError):
raise error.ResponseError(
_('unexpected response from remote server:'), l)
if repo.ui.debugflag:
repo.ui.debug('adding %s (%s)\n' %
(name, util.bytecount(size)))
# for backwards compat, name was partially encoded
ofp = repo.svfs(store.decodedir(name), 'w')
for chunk in util.filechunkiter(fp, limit=size):
handled_bytes += len(chunk)
repo.ui.progress(_('clone'),
handled_bytes,
total=bytecount)
ofp.write(chunk)
ofp.close()
tr.close()
finally:
tr.release()
# Writing straight to files circumvented the inmemory caches
repo.invalidate()
elapsed = time.time() - start
if elapsed <= 0:
elapsed = 0.001
repo.ui.progress(_('clone'), None)
repo.ui.status(
_('transferred %s in %.1f seconds (%s/sec)\n') %
(util.bytecount(bytecount), elapsed,
util.bytecount(bytecount / elapsed)))
finally:
lock.release()
def consumev1(repo, fp, filecount, bytecount):
"""Apply the contents from version 1 of a streaming clone file handle.
This takes the output from "streamout" and applies it to the specified
repository.
Like "streamout," the status line added by the wire protocol is not handled
by this function.
"""
lock = repo.lock()
try:
repo.ui.status(_('%d files to transfer, %s of data\n') %
(filecount, util.bytecount(bytecount)))
handled_bytes = 0
repo.ui.progress(_('clone'), 0, total=bytecount)
start = time.time()
tr = repo.transaction(_('clone'))
try:
for i in xrange(filecount):
# XXX doesn't support '\n' or '\r' in filenames
l = fp.readline()
try:
name, size = l.split('\0', 1)
size = int(size)
except (ValueError, TypeError):
raise error.ResponseError(
_('unexpected response from remote server:'), l)
if repo.ui.debugflag:
repo.ui.debug('adding %s (%s)\n' %
(name, util.bytecount(size)))
# for backwards compat, name was partially encoded
ofp = repo.svfs(store.decodedir(name), 'w')
for chunk in util.filechunkiter(fp, limit=size):
handled_bytes += len(chunk)
repo.ui.progress(_('clone'), handled_bytes, total=bytecount)
ofp.write(chunk)
ofp.close()
tr.close()
finally:
tr.release()
# Writing straight to files circumvented the inmemory caches
repo.invalidate()
elapsed = time.time() - start
if elapsed <= 0:
elapsed = 0.001
repo.ui.progress(_('clone'), None)
repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') %
(util.bytecount(bytecount), elapsed,
util.bytecount(bytecount / elapsed)))
finally:
lock.release()
def _walkstreamfiles(orig, repo):
if state.shallowremote:
# if we are shallow ourselves, stream our local commits
if shallowrepo.requirement in repo.requirements:
striplen = len(repo.store.path) + 1
readdir = repo.store.rawvfs.readdir
visit = [os.path.join(repo.store.path, 'data')]
while visit:
p = visit.pop()
for f, kind, st in readdir(p, stat=True):
fp = p + '/' + f
if kind == stat.S_IFREG:
if not fp.endswith('.i') and not fp.endswith('.d'):
n = util.pconvert(fp[striplen:])
yield (store.decodedir(n), n, st.st_size)
if kind == stat.S_IFDIR:
visit.append(fp)
shallowtrees = repo.ui.configbool('remotefilelog', 'shallowtrees',
False)
if 'treemanifest' in repo.requirements and not shallowtrees:
for (u, e, s) in repo.store.datafiles():
if (u.startswith('meta/') and
(u.endswith('.i') or u.endswith('.d'))):
yield (u, e, s)
# Return .d and .i files that do not match the shallow pattern
match = state.match
if match and not match.always():
for (u, e, s) in repo.store.datafiles():
f = u[5:-2] # trim data/... and .i/.d
if not state.match(f):
yield (u, e, s)
for x in repo.store.topfiles():
if shallowtrees and x[0][:15] == '00manifesttree.':
continue
if state.noflatmf and x[0][:11] == '00manifest.':
continue
yield x
elif shallowrepo.requirement in repo.requirements:
# don't allow cloning from a shallow repo to a full repo
# since it would require fetching every version of every
# file in order to create the revlogs.
raise error.Abort(_("Cannot clone from a shallow repo "
"to a full repo."))
else:
for x in orig(repo):
yield x
def _walkstreamfiles(orig, repo, matcher=None):
if state.shallowremote:
# if we are shallow ourselves, stream our local commits
if shallowutil.isenabled(repo):
striplen = len(repo.store.path) + 1
readdir = repo.store.rawvfs.readdir
visit = [os.path.join(repo.store.path, b'data')]
while visit:
p = visit.pop()
for f, kind, st in readdir(p, stat=True):
fp = p + b'/' + f
if kind == stat.S_IFREG:
if not fp.endswith(b'.i') and not fp.endswith(
b'.d'):
n = util.pconvert(fp[striplen:])
d = store.decodedir(n)
t = store.FILETYPE_OTHER
yield (t, d, n, st.st_size)
if kind == stat.S_IFDIR:
visit.append(fp)
if scmutil.istreemanifest(repo):
for (t, u, e, s) in repo.store.datafiles():
if u.startswith(b'meta/') and (u.endswith(b'.i')
or u.endswith(b'.d')):
yield (t, u, e, s)
# Return .d and .i files that do not match the shallow pattern
match = state.match
if match and not match.always():
for (t, u, e, s) in repo.store.datafiles():
f = u[5:-2] # trim data/... and .i/.d
if not state.match(f):
yield (t, u, e, s)
for x in repo.store.topfiles():
if state.noflatmf and x[1][:11] == b'00manifest.':
continue
yield x
elif shallowutil.isenabled(repo):
# don't allow cloning from a shallow repo to a full repo
# since it would require fetching every version of every
# file in order to create the revlogs.
raise error.Abort(
_(b"Cannot clone from a shallow repo to a full repo."))
else:
for x in orig(repo, matcher):
yield x
def _walk(self, relpath, recurse, allfiles=False):
"""modifies walk to return non .i/.d files so streaming clones
can send remotefilelog store/data files
"""
path = self.path
if relpath:
path += '/' + relpath
striplen = len(self.path) + 1
l = []
if self.rawvfs.isdir(path):
visit = [path]
readdir = self.rawvfs.readdir
while visit:
p = visit.pop()
for f, kind, st in readdir(p, stat=True):
fp = p + '/' + f
if (kind == stat.S_IFREG
and (allfiles or f[-2:] in ('.d', '.i'))):
n = util.pconvert(fp[striplen:])
l.append((storemod.decodedir(n), n, st.st_size))
elif kind == stat.S_IFDIR and recurse:
visit.append(fp)
l.sort()
return l
def _walk(self, relpath, recurse, allfiles=False):
"""modifies walk to return non .i/.d files so streaming clones
can send remotefilelog store/data files
"""
path = self.path
if relpath:
path += '/' + relpath
striplen = len(self.path) + 1
l = []
if self.rawvfs.isdir(path):
visit = [path]
readdir = self.rawvfs.readdir
while visit:
p = visit.pop()
for f, kind, st in readdir(p, stat=True):
fp = p + '/' + f
if (kind == stat.S_IFREG and
(allfiles or f[-2:] in ('.d', '.i'))):
n = util.pconvert(fp[striplen:])
l.append((storemod.decodedir(n), n, st.st_size))
elif kind == stat.S_IFDIR and recurse:
visit.append(fp)
l.sort()
return l