def _getfile(self, tmpfile, filename, hash):
path = lfutil.findfile(self.remote, hash)
if not path:
raise basestore.StoreError(filename, hash, self.url,
_(b"can't get file locally"))
with open(path, b'rb') as fd:
return lfutil.copyandhash(util.filechunkiter(fd), tmpfile)
def drain(self):
'''
Need to read all data from request, httplib is half-duplex
'''
length = int(self.env.get('CONTENT_LENGTH', 0))
for s in util.filechunkiter(self.inp, limit=length):
pass
def download(self, oid, src, content_length):
"""Read the blob from the remote source in chunks, verify the content,
and write to this local blobstore."""
sha256 = hashlib.sha256()
size = 0
with self.vfs(oid, b'wb', atomictemp=True) as fp:
for chunk in util.filechunkiter(src, size=1048576):
fp.write(chunk)
sha256.update(chunk)
size += len(chunk)
# If the server advertised a length longer than what we actually
# received, then we should expect that the server crashed while
# producing the response (but the server has no way of telling us
# that), and we really don't need to try to write the response to
# the localstore, because it's not going to match the expected.
if content_length is not None and int(content_length) != size:
msg = (b"Response length (%s) does not match Content-Length "
b"header (%d): likely server-side crash")
raise LfsRemoteError(_(msg) % (size, int(content_length)))
realoid = hex(sha256.digest())
if realoid != oid:
raise LfsCorruptionError(
_(b'corrupt remote lfs object: %s') % oid)
self._linktousercache(oid)
def drain(self):
'''
Need to read all data from request, httplib is half-duplex
'''
length = int(self.env.get('CONTENT_LENGTH', 0))
for s in util.filechunkiter(self.inp, limit=length):
pass
def snapshot_wdir(ui, repo, files, tmproot):
'''snapshot files from working directory.
if not using snapshot, -I/-X does not work and recursive diff
in tools like kdiff3 and meld displays too many files.'''
repo_root = repo.root
dirname = os.path.basename(repo_root)
if dirname == "":
dirname = "root"
base = os.path.join(tmproot, dirname)
os.mkdir(base)
ui.note(_('making snapshot of %d files from working dir\n') %
(len(files)))
fns_and_mtime = []
for fn in files:
wfn = util.pconvert(fn)
ui.note(' %s\n' % wfn)
dest = os.path.join(base, wfn)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
fp = open(dest, 'wb')
for chunk in util.filechunkiter(repo.wopener(wfn)):
fp.write(chunk)
fp.close()
fns_and_mtime.append((dest, os.path.join(repo_root, fn),
os.path.getmtime(dest)))
return dirname, fns_and_mtime
def hexsha1(data):
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
object data"""
h = util.sha1()
for chunk in util.filechunkiter(data):
h.update(chunk)
return h.hexdigest()
def hexsha1(fileobj):
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
object data"""
h = hashlib.sha1()
for chunk in util.filechunkiter(fileobj):
h.update(chunk)
return hex(h.digest())
def hexsha1(data):
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
object data"""
h = util.sha1()
for chunk in util.filechunkiter(data):
h.update(chunk)
return h.hexdigest()
def consumestreamclone(repo, fp):
"""Apply the contents from a streaming clone file.
This code is copied from Mercurial. Until Mercurial 3.5, this code was
a closure in wireproto.py and not consumeable by extensions.
"""
lock = repo.lock()
try:
repo.ui.status(_('streaming all changes\n'))
l = fp.readline()
try:
total_files, total_bytes = map(int, l.split(' ', 1))
except (ValueError, TypeError):
raise error.ResponseError(
_('unexpected response from remote server:'), l)
repo.ui.status(_('%d files to transfer, %s of data\n') %
(total_files, util.bytecount(total_bytes)))
handled_bytes = 0
repo.ui.progress(_('clone'), 0, total=total_bytes)
start = time.time()
tr = repo.transaction(_('clone'))
try:
for i in xrange(total_files):
# XXX doesn't support '\n' or '\r' in filenames
l = fp.readline()
try:
name, size = l.split('\0', 1)
size = int(size)
except (ValueError, TypeError):
raise error.ResponseError(
_('unexpected response from remote server:'), l)
if repo.ui.debugflag:
repo.ui.debug('adding %s (%s)\n' %
(name, util.bytecount(size)))
# for backwards compat, name was partially encoded
ofp = repo.svfs(store.decodedir(name), 'w')
for chunk in util.filechunkiter(fp, limit=size):
handled_bytes += len(chunk)
repo.ui.progress(_('clone'), handled_bytes,
total=total_bytes)
ofp.write(chunk)
ofp.close()
tr.close()
finally:
tr.release()
# Writing straight to files circumvented the inmemory caches
repo.invalidate()
elapsed = time.time() - start
if elapsed <= 0:
elapsed = 0.001
repo.ui.progress(_('clone'), None)
repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') %
(util.bytecount(total_bytes), elapsed,
util.bytecount(total_bytes / elapsed)))
finally:
lock.release()
def _getfile(self, tmpfile, filename, hash):
path = lfutil.findfile(self.remote, hash)
if not path:
raise basestore.StoreError(filename, hash, self.url,
_("can't get file locally"))
with open(path, 'rb') as fd:
return lfutil.copyandhash(
util.filechunkiter(fd), tmpfile)
def hashfile(file):
if not os.path.exists(file):
return ''
hasher = hashlib.sha1('')
with open(file, 'rb') as fd:
for data in util.filechunkiter(fd):
hasher.update(data)
return hasher.hexdigest()
def hashfile(file):
if not os.path.exists(file):
return ''
hasher = hashlib.sha1('')
with open(file, 'rb') as fd:
for data in util.filechunkiter(fd):
hasher.update(data)
return hasher.hexdigest()
def consumev1(repo, fp, filecount, bytecount):
"""Apply the contents from version 1 of a streaming clone file handle.
This takes the output from "streamout" and applies it to the specified
repository.
Like "streamout," the status line added by the wire protocol is not handled
by this function.
"""
lock = repo.lock()
try:
repo.ui.status(
_('%d files to transfer, %s of data\n') %
(filecount, util.bytecount(bytecount)))
handled_bytes = 0
repo.ui.progress(_('clone'), 0, total=bytecount)
start = time.time()
tr = repo.transaction(_('clone'))
try:
for i in xrange(filecount):
# XXX doesn't support '\n' or '\r' in filenames
l = fp.readline()
try:
name, size = l.split('\0', 1)
size = int(size)
except (ValueError, TypeError):
raise error.ResponseError(
_('unexpected response from remote server:'), l)
if repo.ui.debugflag:
repo.ui.debug('adding %s (%s)\n' %
(name, util.bytecount(size)))
# for backwards compat, name was partially encoded
ofp = repo.svfs(store.decodedir(name), 'w')
for chunk in util.filechunkiter(fp, limit=size):
handled_bytes += len(chunk)
repo.ui.progress(_('clone'),
handled_bytes,
total=bytecount)
ofp.write(chunk)
ofp.close()
tr.close()
finally:
tr.release()
# Writing straight to files circumvented the inmemory caches
repo.invalidate()
elapsed = time.time() - start
if elapsed <= 0:
elapsed = 0.001
repo.ui.progress(_('clone'), None)
repo.ui.status(
_('transferred %s in %.1f seconds (%s/sec)\n') %
(util.bytecount(bytecount), elapsed,
util.bytecount(bytecount / elapsed)))
finally:
lock.release()
def hashfile(file):
if not os.path.exists(file):
return ''
hasher = util.sha1('')
fd = open(file, 'rb')
for data in util.filechunkiter(fd, 128 * 1024):
hasher.update(data)
fd.close()
return hasher.hexdigest()
def hashfile(file):
if not os.path.exists(file):
return ''
hasher = util.sha1('')
fd = open(file, 'rb')
for data in util.filechunkiter(fd, 128 * 1024):
hasher.update(data)
fd.close()
return hasher.hexdigest()
def link(src, dest):
try:
util.oslink(src, dest)
except OSError:
# if hardlinks fail, fallback on atomic copy
dst = util.atomictempfile(dest)
for chunk in util.filechunkiter(open(src, 'rb')):
dst.write(chunk)
dst.close()
os.chmod(dest, os.stat(src).st_mode)
def verify(self, oid):
"""Indicate whether or not the hash of the underlying file matches its
name."""
sha256 = hashlib.sha256()
with self.open(oid) as fp:
for chunk in util.filechunkiter(fp, size=1048576):
sha256.update(chunk)
return oid == sha256.hexdigest()
def link(src, dest):
try:
util.oslink(src, dest)
except OSError:
# if hardlinks fail, fallback on atomic copy
dst = util.atomictempfile(dest)
for chunk in util.filechunkiter(open(src, 'rb')):
dst.write(chunk)
dst.close()
os.chmod(dest, os.stat(src).st_mode)
def link(src, dest):
"""Try to create hardlink - if that fails, efficiently make a copy."""
util.makedirs(os.path.dirname(dest))
try:
util.oslink(src, dest)
except OSError:
# if hardlinks fail, fallback on atomic copy
with open(src, 'rb') as srcf, util.atomictempfile(dest) as dstf:
for chunk in util.filechunkiter(srcf):
dstf.write(chunk)
os.chmod(dest, os.stat(src).st_mode)
def copytostoreabsolute(repo, file, hash):
if inusercache(repo.ui, hash):
link(usercachepath(repo.ui, hash), storepath(repo, hash))
else:
util.makedirs(os.path.dirname(storepath(repo, hash)))
with open(file, 'rb') as srcf:
with util.atomictempfile(storepath(repo, hash),
createmode=repo.store.createmode) as dstf:
for chunk in util.filechunkiter(srcf):
dstf.write(chunk)
linktousercache(repo, hash)
def copytostoreabsolute(repo, file, hash):
if inusercache(repo.ui, hash):
link(usercachepath(repo.ui, hash), storepath(repo, hash))
else:
util.makedirs(os.path.dirname(storepath(repo, hash)))
with open(file, 'rb') as srcf:
with util.atomictempfile(storepath(repo, hash),
createmode=repo.store.createmode) as dstf:
for chunk in util.filechunkiter(srcf):
dstf.write(chunk)
linktousercache(repo, hash)
def copytostoreabsolute(repo, file, hash):
util.makedirs(os.path.dirname(storepath(repo, hash)))
if inusercache(repo.ui, hash):
link(usercachepath(repo.ui, hash), storepath(repo, hash))
else:
dst = util.atomictempfile(storepath(repo, hash))
for chunk in util.filechunkiter(open(file, 'rb')):
dst.write(chunk)
dst.close()
util.copymode(file, storepath(repo, hash))
linktousercache(repo, hash)
def consumev1(repo, fp, filecount, bytecount):
"""Apply the contents from version 1 of a streaming clone file handle.
This takes the output from "streamout" and applies it to the specified
repository.
Like "streamout," the status line added by the wire protocol is not handled
by this function.
"""
lock = repo.lock()
try:
repo.ui.status(_('%d files to transfer, %s of data\n') %
(filecount, util.bytecount(bytecount)))
handled_bytes = 0
repo.ui.progress(_('clone'), 0, total=bytecount)
start = time.time()
tr = repo.transaction(_('clone'))
try:
for i in xrange(filecount):
# XXX doesn't support '\n' or '\r' in filenames
l = fp.readline()
try:
name, size = l.split('\0', 1)
size = int(size)
except (ValueError, TypeError):
raise error.ResponseError(
_('unexpected response from remote server:'), l)
if repo.ui.debugflag:
repo.ui.debug('adding %s (%s)\n' %
(name, util.bytecount(size)))
# for backwards compat, name was partially encoded
ofp = repo.svfs(store.decodedir(name), 'w')
for chunk in util.filechunkiter(fp, limit=size):
handled_bytes += len(chunk)
repo.ui.progress(_('clone'), handled_bytes, total=bytecount)
ofp.write(chunk)
ofp.close()
tr.close()
finally:
tr.release()
# Writing straight to files circumvented the inmemory caches
repo.invalidate()
elapsed = time.time() - start
if elapsed <= 0:
elapsed = 0.001
repo.ui.progress(_('clone'), None)
repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') %
(util.bytecount(bytecount), elapsed,
util.bytecount(bytecount / elapsed)))
finally:
lock.release()
def copytostoreabsolute(repo, file, hash):
if inusercache(repo.ui, hash):
link(usercachepath(repo.ui, hash), storepath(repo, hash))
elif not getattr(repo, "_isconverting", False):
util.makedirs(os.path.dirname(storepath(repo, hash)))
dst = util.atomictempfile(storepath(repo, hash),
createmode=repo.store.createmode)
for chunk in util.filechunkiter(open(file, 'rb')):
dst.write(chunk)
dst.close()
linktousercache(repo, hash)
def copytostoreabsolute(repo, file, hash):
if inusercache(repo.ui, hash):
link(usercachepath(repo.ui, hash), storepath(repo, hash))
else:
util.makedirs(os.path.dirname(storepath(repo, hash)))
dst = util.atomictempfile(storepath(repo, hash),
createmode=repo.store.createmode)
for chunk in util.filechunkiter(open(file, 'rb')):
dst.write(chunk)
dst.close()
linktousercache(repo, hash)
def link(src, dest):
"""Try to create hardlink - if that fails, efficiently make a copy."""
util.makedirs(os.path.dirname(dest))
try:
util.oslink(src, dest)
except OSError:
# if hardlinks fail, fallback on atomic copy
dst = util.atomictempfile(dest)
for chunk in util.filechunkiter(open(src, 'rb')):
dst.write(chunk)
dst.close()
os.chmod(dest, os.stat(src).st_mode)
def bail(response, headers={}):
length = int(req.env.get('CONTENT_LENGTH', 0))
for s in util.filechunkiter(req, limit=length):
# drain incoming bundle, else client will not see
# response when run outside cgi script
pass
status = headers.pop('status', HTTP_OK)
req.header(headers.items())
req.respond(status, HGTYPE)
req.write('0\n')
req.write(response)
def overridecat(orig, ui, repo, file1, *pats, **opts):
ctx = scmutil.revsingle(repo, opts.get('rev'))
err = 1
notbad = set()
m = scmutil.match(ctx, (file1, ) + pats, opts)
origmatchfn = m.matchfn
def lfmatchfn(f):
if origmatchfn(f):
return True
lf = lfutil.splitstandin(f)
if lf is None:
return False
notbad.add(lf)
return origmatchfn(lf)
m.matchfn = lfmatchfn
origbadfn = m.bad
def lfbadfn(f, msg):
if not f in notbad:
origbadfn(f, msg)
m.bad = lfbadfn
for f in ctx.walk(m):
fp = cmdutil.makefileobj(repo,
opts.get('output'),
ctx.node(),
pathname=f)
lf = lfutil.splitstandin(f)
if lf is None or origmatchfn(f):
# duplicating unreachable code from commands.cat
data = ctx[f].data()
if opts.get('decode'):
data = repo.wwritedata(f, data)
fp.write(data)
else:
hash = lfutil.readstandin(repo, lf, ctx.rev())
if not lfutil.inusercache(repo.ui, hash):
store = basestore._openstore(repo)
success, missing = store.get([(lf, hash)])
if len(success) != 1:
raise util.Abort(
_('largefile %s is not in cache and could not be '
'downloaded') % lf)
path = lfutil.usercachepath(repo.ui, hash)
fpin = open(path, "rb")
for chunk in util.filechunkiter(fpin, 128 * 1024):
fp.write(chunk)
fpin.close()
fp.close()
err = 0
return err
def generatestreamclone(repo):
"""Emit content for a streaming clone.
This is a generator of raw chunks that constitute a streaming clone.
This code is copied from Mercurial. Until Mercurial 3.5, this code was
a closure in wireproto.py and not consumeable by extensions.
"""
entries = []
total_bytes = 0
# Get consistent snapshot of repo, lock during scan.
lock = repo.lock()
try:
repo.ui.debug('scanning\n')
for name, ename, size in repo.store.walk():
if size:
entries.append((name, size))
total_bytes += size
finally:
lock.release()
repo.ui.debug('%d files, %d bytes to transfer\n' %
(len(entries), total_bytes))
yield '%d %d\n' % (len(entries), total_bytes)
sopener = repo.svfs
oldaudit = sopener.mustaudit
debugflag = repo.ui.debugflag
sopener.mustaudit = False
try:
for name, size in entries:
if debugflag:
repo.ui.debug('sending %s (%d bytes)\n' % (name, size))
# partially encode name over the wire for backwards compat
yield '%s\0%d\n' % (store.encodedir(name), size)
if size <= 65536:
fp = sopener(name)
try:
data = fp.read(size)
finally:
fp.close()
yield data
else:
for chunk in util.filechunkiter(sopener(name), limit=size):
yield chunk
finally:
sopener.mustaudit = oldaudit
def download(self, oid, src):
"""Read the blob from the remote source in chunks, verify the content,
and write to this local blobstore."""
sha256 = hashlib.sha256()
with self.vfs(oid, 'wb', atomictemp=True) as fp:
for chunk in util.filechunkiter(src, size=1048576):
fp.write(chunk)
sha256.update(chunk)
realoid = sha256.hexdigest()
if realoid != oid:
raise LfsCorruptionError(_('corrupt remote lfs object: %s')
% oid)
self._linktousercache(oid)
def overridecat(orig, ui, repo, file1, *pats, **opts):
ctx = scmutil.revsingle(repo, opts.get('rev'))
err = 1
notbad = set()
m = scmutil.match(ctx, (file1,) + pats, opts)
origmatchfn = m.matchfn
def lfmatchfn(f):
if origmatchfn(f):
return True
lf = lfutil.splitstandin(f)
if lf is None:
return False
notbad.add(lf)
return origmatchfn(lf)
m.matchfn = lfmatchfn
origbadfn = m.bad
def lfbadfn(f, msg):
if not f in notbad:
origbadfn(f, msg)
m.bad = lfbadfn
for f in ctx.walk(m):
fp = cmdutil.makefileobj(repo, opts.get('output'), ctx.node(),
pathname=f)
lf = lfutil.splitstandin(f)
if lf is None or origmatchfn(f):
# duplicating unreachable code from commands.cat
data = ctx[f].data()
if opts.get('decode'):
data = repo.wwritedata(f, data)
fp.write(data)
else:
hash = lfutil.readstandin(repo, lf, ctx.rev())
if not lfutil.inusercache(repo.ui, hash):
store = basestore._openstore(repo)
success, missing = store.get([(lf, hash)])
if len(success) != 1:
raise util.Abort(
_('largefile %s is not in cache and could not be '
'downloaded') % lf)
path = lfutil.usercachepath(repo.ui, hash)
fpin = open(path, "rb")
for chunk in util.filechunkiter(fpin, 128 * 1024):
fp.write(chunk)
fpin.close()
fp.close()
err = 0
return err
def getlfile(self, sha):
"""returns an iterable with the chunks of the file with sha sha"""
stream = self._callstream("getlfile", sha=sha)
length = stream.readline()
try:
length = int(length)
except ValueError:
self._abort(error.ResponseError(_("unexpected response:"), length))
# SSH streams will block if reading more than length
for chunk in util.filechunkiter(stream, 128 * 1024, length):
yield chunk
# HTTP streams must hit the end to process the last empty
# chunk of Chunked-Encoding so the connection can be reused.
if issubclass(self.__class__, httppeer.httppeer):
chunk = stream.read(1)
if chunk:
self._abort(error.ResponseError(_("unexpected response:"), chunk))
def emitrevlogdata():
try:
for name, size in entries:
if debugflag:
repo.ui.debug('sending %s (%d bytes)\n' % (name, size))
# partially encode name over the wire for backwards compat
yield '%s\0%d\n' % (store.encodedir(name), size)
if size <= 65536:
fp = svfs(name)
try:
data = fp.read(size)
finally:
fp.close()
yield data
else:
for chunk in util.filechunkiter(svfs(name), limit=size):
yield chunk
finally:
svfs.mustaudit = oldaudit
def emitrevlogdata():
try:
for name, size in entries:
if debugflag:
repo.ui.debug('sending %s (%d bytes)\n' % (name, size))
# partially encode name over the wire for backwards compat
yield '%s\0%d\n' % (store.encodedir(name), size)
if size <= 65536:
fp = svfs(name)
try:
data = fp.read(size)
finally:
fp.close()
yield data
else:
for chunk in util.filechunkiter(svfs(name), limit=size):
yield chunk
finally:
svfs.mustaudit = oldaudit
def download(self, oid, src):
"""Read the blob from the remote source in chunks, verify the content,
and write to this local blobstore."""
sha256 = hashlib.sha256()
with self.vfs(oid, 'wb', atomictemp=True) as fp:
for chunk in util.filechunkiter(src, size=1048576):
fp.write(chunk)
sha256.update(chunk)
realoid = sha256.hexdigest()
if realoid != oid:
raise error.Abort(_('corrupt remote lfs object: %s') % oid)
# XXX: should we verify the content of the cache, and hardlink back to
# the local store on success, but truncate, write and link on failure?
if not self.cachevfs.exists(oid):
self.ui.note(_('lfs: adding %s to the usercache\n') % oid)
lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid))
def getlfile(self, sha):
"""returns an iterable with the chunks of the file with sha sha"""
stream = self._callstream("getlfile", sha=sha)
length = stream.readline()
try:
length = int(length)
except ValueError:
self._abort(
error.ResponseError(_("unexpected response:"), length))
# SSH streams will block if reading more than length
for chunk in util.filechunkiter(stream, limit=length):
yield chunk
# HTTP streams must hit the end to process the last empty
# chunk of Chunked-Encoding so the connection can be reused.
if issubclass(self.__class__, httppeer.httppeer):
chunk = stream.read(1)
if chunk:
self._abort(
error.ResponseError(_("unexpected response:"), chunk))
def copyfromcache(repo, hash, filename):
'''Copy the specified largefile from the repo or system cache to
filename in the repository. Return true on success or false if the
file was not found in either cache (which should not happened:
this is meant to be called only after ensuring that the needed
largefile exists in the cache).'''
wvfs = repo.wvfs
path = findfile(repo, hash)
if path is None:
return False
wvfs.makedirs(wvfs.dirname(wvfs.join(filename)))
# The write may fail before the file is fully written, but we
# don't use atomic writes in the working copy.
with open(path, 'rb') as srcfd, wvfs(filename, 'wb') as destfd:
gothash = copyandhash(
util.filechunkiter(srcfd), destfd)
if gothash != hash:
repo.ui.warn(_('%s: data corruption in %s with hash %s\n')
% (filename, path, gothash))
wvfs.unlink(filename)
return False
return True
def copyfromcache(repo, hash, filename):
'''Copy the specified largefile from the repo or system cache to
filename in the repository. Return true on success or false if the
file was not found in either cache (which should not happened:
this is meant to be called only after ensuring that the needed
largefile exists in the cache).'''
wvfs = repo.wvfs
path = findfile(repo, hash)
if path is None:
return False
wvfs.makedirs(wvfs.dirname(wvfs.join(filename)))
# The write may fail before the file is fully written, but we
# don't use atomic writes in the working copy.
with open(path, 'rb') as srcfd:
with wvfs(filename, 'wb') as destfd:
gothash = copyandhash(
util.filechunkiter(srcfd), destfd)
if gothash != hash:
repo.ui.warn(_('%s: data corruption in %s with hash %s\n')
% (filename, path, gothash))
wvfs.unlink(filename)
return False
return True
def _docensor(ui, repo, path, rev='', tombstone='', **opts):
if not path:
raise error.Abort(_('must specify file path to censor'))
if not rev:
raise error.Abort(_('must specify revision to censor'))
wctx = repo[None]
m = scmutil.match(wctx, (path,))
if m.anypats() or len(m.files()) != 1:
raise error.Abort(_('can only specify an explicit filename'))
path = m.files()[0]
flog = repo.file(path)
if not len(flog):
raise error.Abort(_('cannot censor file with no history'))
rev = scmutil.revsingle(repo, rev, rev).rev()
try:
ctx = repo[rev]
except KeyError:
raise error.Abort(_('invalid revision identifier %s') % rev)
try:
fctx = ctx.filectx(path)
except error.LookupError:
raise error.Abort(_('file does not exist at revision %s') % rev)
fnode = fctx.filenode()
headctxs = [repo[c] for c in repo.heads()]
heads = [c for c in headctxs if path in c and c.filenode(path) == fnode]
if heads:
headlist = ', '.join([short(c.node()) for c in heads])
raise error.Abort(_('cannot censor file in heads (%s)') % headlist,
hint=_('clean/delete and commit first'))
wp = wctx.parents()
if ctx.node() in [p.node() for p in wp]:
raise error.Abort(_('cannot censor working directory'),
hint=_('clean/delete/update first'))
flogv = flog.version & 0xFFFF
if flogv != revlog.REVLOGNG:
raise error.Abort(
_('censor does not support revlog version %d') % (flogv,))
tombstone = filelog.packmeta({"censored": tombstone}, "")
crev = fctx.filerev()
if len(tombstone) > flog.rawsize(crev):
raise error.Abort(_(
'censor tombstone must be no longer than censored data'))
# Using two files instead of one makes it easy to rewrite entry-by-entry
idxread = repo.svfs(flog.indexfile, 'r')
idxwrite = repo.svfs(flog.indexfile, 'wb', atomictemp=True)
if flog.version & revlog.REVLOGNGINLINEDATA:
dataread, datawrite = idxread, idxwrite
else:
dataread = repo.svfs(flog.datafile, 'r')
datawrite = repo.svfs(flog.datafile, 'wb', atomictemp=True)
# Copy all revlog data up to the entry to be censored.
rio = revlog.revlogio()
offset = flog.start(crev)
for chunk in util.filechunkiter(idxread, limit=crev * rio.size):
idxwrite.write(chunk)
for chunk in util.filechunkiter(dataread, limit=offset):
datawrite.write(chunk)
def rewriteindex(r, newoffs, newdata=None):
"""Rewrite the index entry with a new data offset and optional new data.
The newdata argument, if given, is a tuple of three positive integers:
(new compressed, new uncompressed, added flag bits).
"""
offlags, comp, uncomp, base, link, p1, p2, nodeid = flog.index[r]
flags = revlog.gettype(offlags)
if newdata:
comp, uncomp, nflags = newdata
flags |= nflags
offlags = revlog.offset_type(newoffs, flags)
e = (offlags, comp, uncomp, r, link, p1, p2, nodeid)
idxwrite.write(rio.packentry(e, None, flog.version, r))
idxread.seek(rio.size, 1)
def rewrite(r, offs, data, nflags=revlog.REVIDX_DEFAULT_FLAGS):
"""Write the given full text to the filelog with the given data offset.
Returns:
The integer number of data bytes written, for tracking data offsets.
"""
flag, compdata = flog.compress(data)
newcomp = len(flag) + len(compdata)
rewriteindex(r, offs, (newcomp, len(data), nflags))
datawrite.write(flag)
datawrite.write(compdata)
dataread.seek(flog.length(r), 1)
return newcomp
# Rewrite censored revlog entry with (padded) tombstone data.
pad = ' ' * (flog.rawsize(crev) - len(tombstone))
offset += rewrite(crev, offset, tombstone + pad, revlog.REVIDX_ISCENSORED)
# Rewrite all following filelog revisions fixing up offsets and deltas.
for srev in xrange(crev + 1, len(flog)):
if crev in flog.parentrevs(srev):
# Immediate children of censored node must be re-added as fulltext.
try:
revdata = flog.revision(srev)
except error.CensoredNodeError as e:
revdata = e.tombstone
dlen = rewrite(srev, offset, revdata)
else:
# Copy any other revision data verbatim after fixing up the offset.
rewriteindex(srev, offset)
dlen = flog.length(srev)
for chunk in util.filechunkiter(dataread, limit=dlen):
datawrite.write(chunk)
offset += dlen
idxread.close()
idxwrite.close()
if dataread is not idxread:
dataread.close()
datawrite.close()
def unbundle(repo, req):
"""
Because protocol.unbundle uses tempfile, we cannot use it in
AppEngine environment. Here is an alternative implementation
which does not use tempfiles.
"""
proto = os.environ.get("wsgi.url_scheme") or "http"
their_heads = req.form["heads"][0].split(" ")
def check_heads():
heads = map(hex, repo.heads())
return their_heads == [hex("force")] or their_heads == heads
# fail early if possible
if not check_heads():
length = int(req.headers.get("Content-Length", 0))
for s in util.filechunkiter(req.body_file, limit=length):
# drain incoming bundle, else client will not see
# response when run outside cgi script
pass
raise ErrorResponse(HTTP_OK, "unsynced changes")
# do not lock repo until all changegroup data is
# streamed. save to temporary file.
fp = StringIO()
length = int(req.headers["Content-Length"])
if int(length) > 50 * 1024:
# Content is too long:
raise ErrorResponse(HTTP_OK, "Bundled change size is greater than 50k." " Split changes using -r flag.")
for s in util.filechunkiter(req.body_file, limit=length):
fp.write(s)
try:
if not check_heads():
raise ErrorResponse(HTTP_OK, "unsynced changes")
fp.seek(0)
header = fp.read(6)
if header.startswith("HG") and not header.startswith("HG10"):
raise ValueError("unknown bundle version")
elif header not in changegroup.bundletypes:
raise ValueError("unknown bundle compression type")
gen = changegroup.unbundle(header, fp)
# send addchangegroup output to client
oldio = sys.stdout, sys.stderr
sys.stderr = sys.stdout = StringIO()
try:
url = "remote:%s:%s" % (proto, req.remote_addr)
try:
ret = repo.addchangegroup(gen, "serve", url)
except util.Abort, inst:
sys.stdout.write("abort: %s\n" % inst)
ret = 0
finally:
val = sys.stdout.getvalue()
sys.stdout, sys.stderr = oldio
req.respond(HTTP_OK, protocol.HGTYPE)
return ("%d\n%s" % (ret, val),)
except ValueError, inst:
raise ErrorResponse(HTTP_OK, inst)
def getfile(self, fp):
length = int(self.req.env["CONTENT_LENGTH"])
for s in util.filechunkiter(self.req, limit=length):
fp.write(s)
def generator():
yield '%d\n' % length
for chunk in util.filechunkiter(f):
yield chunk
def unbundle(repo, req):
proto = req.env.get('wsgi.url_scheme') or 'http'
their_heads = req.form['heads'][0].split(' ')
def check_heads():
heads = map(hex, repo.heads())
return their_heads == [hex('force')] or their_heads == heads
# fail early if possible
if not check_heads():
req.drain()
raise ErrorResponse(HTTP_OK, 'unsynced changes')
# do not lock repo until all changegroup data is
# streamed. save to temporary file.
fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-')
fp = os.fdopen(fd, 'wb+')
try:
length = int(req.env['CONTENT_LENGTH'])
for s in util.filechunkiter(req, limit=length):
fp.write(s)
try:
lock = repo.lock()
try:
if not check_heads():
raise ErrorResponse(HTTP_OK, 'unsynced changes')
fp.seek(0)
header = fp.read(6)
if header.startswith('HG') and not header.startswith('HG10'):
raise ValueError('unknown bundle version')
elif header not in changegroupmod.bundletypes:
raise ValueError('unknown bundle compression type')
gen = changegroupmod.unbundle(header, fp)
# send addchangegroup output to client
oldio = sys.stdout, sys.stderr
sys.stderr = sys.stdout = cStringIO.StringIO()
try:
url = 'remote:%s:%s:%s' % (
proto, urllib.quote(req.env.get('REMOTE_HOST', '')),
urllib.quote(req.env.get('REMOTE_USER', '')))
try:
ret = repo.addchangegroup(gen, 'serve', url)
except util.Abort, inst:
sys.stdout.write("abort: %s\n" % inst)
ret = 0
finally:
val = sys.stdout.getvalue()
sys.stdout, sys.stderr = oldio
req.respond(HTTP_OK, HGTYPE)
return '%d\n%s' % (ret, val),
finally:
lock.release()
except ValueError, inst:
raise ErrorResponse(HTTP_OK, inst)
except (OSError, IOError), inst:
filename = getattr(inst, 'filename', '')
# Don't send our filesystem layout to the client
if filename.startswith(repo.root):
filename = filename[len(repo.root) + 1:]
else:
filename = ''
error = getattr(inst, 'strerror', 'Unknown error')
if inst.errno == errno.ENOENT:
code = HTTP_NOT_FOUND
else:
code = HTTP_SERVER_ERROR
raise ErrorResponse(code, '%s: %s' % (error, filename))
def generator():
yield '%d\n' % length
for chunk in util.filechunkiter(f):
yield chunk
def unbundle(web, req):
def bail(response, headers={}):
length = int(req.env.get('CONTENT_LENGTH', 0))
for s in util.filechunkiter(req, limit=length):
# drain incoming bundle, else client will not see
# response when run outside cgi script
pass
status = headers.pop('status', HTTP_OK)
req.header(headers.items())
req.respond(status, HGTYPE)
req.write('0\n')
req.write(response)
# enforce that you can only unbundle with POST requests
if req.env['REQUEST_METHOD'] != 'POST':
headers = {'status': '405 Method Not Allowed'}
bail('unbundle requires POST request\n', headers)
return
# require ssl by default, auth info cannot be sniffed and
# replayed
ssl_req = web.configbool('web', 'push_ssl', True)
if ssl_req:
if req.env.get('wsgi.url_scheme') != 'https':
bail('ssl required\n')
return
proto = 'https'
else:
proto = 'http'
# do not allow push unless explicitly allowed
if not web.check_perm(req, 'push', False):
bail('push not authorized\n', headers={'status': '401 Unauthorized'})
return
their_heads = req.form['heads'][0].split(' ')
def check_heads():
heads = map(hex, web.repo.heads())
return their_heads == [hex('force')] or their_heads == heads
# fail early if possible
if not check_heads():
bail('unsynced changes\n')
return
req.respond(HTTP_OK, HGTYPE)
# do not lock repo until all changegroup data is
# streamed. save to temporary file.
fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-')
fp = os.fdopen(fd, 'wb+')
try:
length = int(req.env['CONTENT_LENGTH'])
for s in util.filechunkiter(req, limit=length):
fp.write(s)
try:
lock = web.repo.lock()
try:
if not check_heads():
req.write('0\n')
req.write('unsynced changes\n')
return
fp.seek(0)
header = fp.read(6)
if header.startswith('HG') and not header.startswith('HG10'):
raise ValueError('unknown bundle version')
elif header not in changegroupmod.bundletypes:
raise ValueError('unknown bundle compression type')
gen = changegroupmod.unbundle(header, fp)
# send addchangegroup output to client
oldio = sys.stdout, sys.stderr
sys.stderr = sys.stdout = cStringIO.StringIO()
try:
url = 'remote:%s:%s' % (proto,
req.env.get('REMOTE_HOST', ''))
try:
ret = web.repo.addchangegroup(gen, 'serve', url)
except util.Abort, inst:
sys.stdout.write("abort: %s\n" % inst)
ret = 0
finally:
val = sys.stdout.getvalue()
sys.stdout, sys.stderr = oldio
req.write('%d\n' % ret)
req.write(val)
finally:
del lock
except ValueError, inst:
req.write('0\n')
req.write(str(inst) + '\n')
except (OSError, IOError), inst:
req.write('0\n')
filename = getattr(inst, 'filename', '')
# Don't send our filesystem layout to the client
if filename.startswith(web.repo.root):
filename = filename[len(web.repo.root)+1:]
else:
filename = ''
error = getattr(inst, 'strerror', 'Unknown error')
if inst.errno == errno.ENOENT:
code = HTTP_NOT_FOUND
else:
code = HTTP_SERVER_ERROR
req.respond(code)
req.write('%s: %s\n' % (error, filename))
def unbundle(repo, req):
proto = req.env.get('wsgi.url_scheme') or 'http'
their_heads = req.form['heads'][0].split(' ')
def check_heads():
heads = map(hex, repo.heads())
return their_heads == [hex('force')] or their_heads == heads
# fail early if possible
if not check_heads():
req.drain()
raise ErrorResponse(HTTP_OK, 'unsynced changes')
# do not lock repo until all changegroup data is
# streamed. save to temporary file.
fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-')
fp = os.fdopen(fd, 'wb+')
try:
length = int(req.env['CONTENT_LENGTH'])
for s in util.filechunkiter(req, limit=length):
fp.write(s)
try:
lock = repo.lock()
try:
if not check_heads():
raise ErrorResponse(HTTP_OK, 'unsynced changes')
fp.seek(0)
header = fp.read(6)
if header.startswith('HG') and not header.startswith('HG10'):
raise ValueError('unknown bundle version')
elif header not in changegroupmod.bundletypes:
raise ValueError('unknown bundle compression type')
gen = changegroupmod.unbundle(header, fp)
# send addchangegroup output to client
oldio = sys.stdout, sys.stderr
sys.stderr = sys.stdout = cStringIO.StringIO()
try:
url = 'remote:%s:%s:%s' % (
proto,
urllib.quote(req.env.get('REMOTE_HOST', '')),
urllib.quote(req.env.get('REMOTE_USER', '')))
try:
ret = repo.addchangegroup(gen, 'serve', url)
except util.Abort, inst:
sys.stdout.write("abort: %s\n" % inst)
ret = 0
finally:
val = sys.stdout.getvalue()
sys.stdout, sys.stderr = oldio
req.respond(HTTP_OK, HGTYPE)
return '%d\n%s' % (ret, val),
finally:
lock.release()
except ValueError, inst:
raise ErrorResponse(HTTP_OK, inst)
except (OSError, IOError), inst:
filename = getattr(inst, 'filename', '')
# Don't send our filesystem layout to the client
if filename.startswith(repo.root):
filename = filename[len(repo.root)+1:]
else:
filename = ''
error = getattr(inst, 'strerror', 'Unknown error')
if inst.errno == errno.ENOENT:
code = HTTP_NOT_FOUND
else:
code = HTTP_SERVER_ERROR
raise ErrorResponse(code, '%s: %s' % (error, filename))
def censor(ui, repo, path, rev='', tombstone='', **opts):
if not path:
raise util.Abort(_('must specify file path to censor'))
if not rev:
raise util.Abort(_('must specify revision to censor'))
wctx = repo[None]
m = scmutil.match(wctx, (path,))
if m.anypats() or len(m.files()) != 1:
raise util.Abort(_('can only specify an explicit filename'))
path = m.files()[0]
flog = repo.file(path)
if not len(flog):
raise util.Abort(_('cannot censor file with no history'))
rev = scmutil.revsingle(repo, rev, rev).rev()
try:
ctx = repo[rev]
except KeyError:
raise util.Abort(_('invalid revision identifier %s') % rev)
try:
fctx = ctx.filectx(path)
except error.LookupError:
raise util.Abort(_('file does not exist at revision %s') % rev)
fnode = fctx.filenode()
headctxs = [repo[c] for c in repo.heads()]
heads = [c for c in headctxs if path in c and c.filenode(path) == fnode]
if heads:
headlist = ', '.join([short(c.node()) for c in heads])
raise util.Abort(_('cannot censor file in heads (%s)') % headlist,
hint=_('clean/delete and commit first'))
wp = wctx.parents()
if ctx.node() in [p.node() for p in wp]:
raise util.Abort(_('cannot censor working directory'),
hint=_('clean/delete/update first'))
flogv = flog.version & 0xFFFF
if flogv != revlog.REVLOGNG:
raise util.Abort(
_('censor does not support revlog version %d') % (flogv,))
tombstone = filelog.packmeta({"censored": tombstone}, "")
crev = fctx.filerev()
if len(tombstone) > flog.rawsize(crev):
raise util.Abort(_(
'censor tombstone must be no longer than censored data'))
# Using two files instead of one makes it easy to rewrite entry-by-entry
idxread = repo.svfs(flog.indexfile, 'r')
idxwrite = repo.svfs(flog.indexfile, 'wb', atomictemp=True)
if flog.version & revlog.REVLOGNGINLINEDATA:
dataread, datawrite = idxread, idxwrite
else:
dataread = repo.svfs(flog.datafile, 'r')
datawrite = repo.svfs(flog.datafile, 'wb', atomictemp=True)
# Copy all revlog data up to the entry to be censored.
rio = revlog.revlogio()
offset = flog.start(crev)
for chunk in util.filechunkiter(idxread, limit=crev * rio.size):
idxwrite.write(chunk)
for chunk in util.filechunkiter(dataread, limit=offset):
datawrite.write(chunk)
def rewriteindex(r, newoffs, newdata=None):
"""Rewrite the index entry with a new data offset and optional new data.
The newdata argument, if given, is a tuple of three positive integers:
(new compressed, new uncompressed, added flag bits).
"""
offlags, comp, uncomp, base, link, p1, p2, nodeid = flog.index[r]
flags = revlog.gettype(offlags)
if newdata:
comp, uncomp, nflags = newdata
flags |= nflags
offlags = revlog.offset_type(newoffs, flags)
e = (offlags, comp, uncomp, r, link, p1, p2, nodeid)
idxwrite.write(rio.packentry(e, None, flog.version, r))
idxread.seek(rio.size, 1)
def rewrite(r, offs, data, nflags=revlog.REVIDX_DEFAULT_FLAGS):
"""Write the given full text to the filelog with the given data offset.
Returns:
The integer number of data bytes written, for tracking data offsets.
"""
flag, compdata = flog.compress(data)
newcomp = len(flag) + len(compdata)
rewriteindex(r, offs, (newcomp, len(data), nflags))
datawrite.write(flag)
datawrite.write(compdata)
dataread.seek(flog.length(r), 1)
return newcomp
# Rewrite censored revlog entry with (padded) tombstone data.
pad = ' ' * (flog.rawsize(crev) - len(tombstone))
offset += rewrite(crev, offset, tombstone + pad, revlog.REVIDX_ISCENSORED)
# Rewrite all following filelog revisions fixing up offsets and deltas.
for srev in xrange(crev + 1, len(flog)):
if crev in flog.parentrevs(srev):
# Immediate children of censored node must be re-added as fulltext.
try:
revdata = flog.revision(srev)
except error.CensoredNodeError as e:
revdata = e.tombstone
dlen = rewrite(srev, offset, revdata)
else:
# Copy any other revision data verbatim after fixing up the offset.
rewriteindex(srev, offset)
dlen = flog.length(srev)
for chunk in util.filechunkiter(dataread, limit=dlen):
datawrite.write(chunk)
offset += dlen
idxread.close()
idxwrite.close()
if dataread is not idxread:
dataread.close()
datawrite.close()
def getfile(self, fp):
length = int(self.req.env['CONTENT_LENGTH'])
for s in util.filechunkiter(self.req, limit=length):
fp.write(s)