def censorrevision(self, tr, censornode, tombstone=b''):
tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
# This restriction is cargo culted from revlogs and makes no sense for
# SQLite, since columns can be resized at will.
if len(tombstone) > len(self.rawdata(censornode)):
raise error.Abort(
_(b'censor tombstone must be no longer than censored data')
)
# We need to replace the censored revision's data with the tombstone.
# But replacing that data will have implications for delta chains that
# reference it.
#
# While "better," more complex strategies are possible, we do something
# simple: we find delta chain children of the censored revision and we
# replace those incremental deltas with fulltexts of their corresponding
# revision. Then we delete the now-unreferenced delta and original
# revision and insert a replacement.
# Find the delta to be censored.
censoreddeltaid = self._db.execute(
'SELECT deltaid FROM fileindex WHERE id=?',
(self._revisions[censornode].rid,),
).fetchone()[0]
# Find all its delta chain children.
# TODO once we support storing deltas for !files, we'll need to look
# for those delta chains too.
rows = list(
self._db.execute(
'SELECT id, pathid, node FROM fileindex '
'WHERE deltabaseid=? OR deltaid=?',
(censoreddeltaid, censoreddeltaid),
)
)
for row in rows:
rid, pathid, node = row
fulltext = resolvedeltachain(
self._db, pathid, node, {}, {-1: None}, zstddctx=self._dctx
)
deltahash = hashutil.sha1(fulltext).digest()
if self._compengine == b'zstd':
deltablob = self._cctx.compress(fulltext)
compression = COMPRESSION_ZSTD
elif self._compengine == b'zlib':
deltablob = zlib.compress(fulltext)
compression = COMPRESSION_ZLIB
elif self._compengine == b'none':
deltablob = fulltext
compression = COMPRESSION_NONE
else:
raise error.ProgrammingError(
b'unhandled compression engine: %s' % self._compengine
)
if len(deltablob) >= len(fulltext):
deltablob = fulltext
compression = COMPRESSION_NONE
deltaid = insertdelta(self._db, compression, deltahash, deltablob)
self._db.execute(
'UPDATE fileindex SET deltaid=?, deltabaseid=NULL '
'WHERE id=?',
(deltaid, rid),
)
# Now create the tombstone delta and replace the delta on the censored
# node.
deltahash = hashutil.sha1(tombstone).digest()
tombstonedeltaid = insertdelta(
self._db, COMPRESSION_NONE, deltahash, tombstone
)
flags = self._revisions[censornode].flags
flags |= FLAG_CENSORED
self._db.execute(
'UPDATE fileindex SET flags=?, deltaid=?, deltabaseid=NULL '
'WHERE pathid=? AND node=?',
(flags, tombstonedeltaid, self._pathid, censornode),
)
self._db.execute('DELETE FROM delta WHERE id=?', (censoreddeltaid,))
self._refreshindex()
self._revisioncache.clear()
def _addrawrevision(
self,
node,
revisiondata,
transaction,
linkrev,
p1,
p2,
storedelta=None,
flags=0,
):
if self._pathid is None:
res = self._db.execute(
'INSERT INTO filepath (path) VALUES (?)', (self._path,)
)
self._pathid = res.lastrowid
# For simplicity, always store a delta against p1.
# TODO we need a lot more logic here to make behavior reasonable.
if storedelta:
deltabase, delta = storedelta
if isinstance(deltabase, int):
deltabase = self.node(deltabase)
else:
assert revisiondata is not None
deltabase = p1
if deltabase == nullid:
delta = revisiondata
else:
delta = mdiff.textdiff(
self.revision(self.rev(deltabase)), revisiondata
)
# File index stores a pointer to its delta and the parent delta.
# The parent delta is stored via a pointer to the fileindex PK.
if deltabase == nullid:
baseid = None
else:
baseid = self._revisions[deltabase].rid
# Deltas are stored with a hash of their content. This allows
# us to de-duplicate. The table is configured to ignore conflicts
# and it is faster to just insert and silently noop than to look
# first.
deltahash = hashutil.sha1(delta).digest()
if self._compengine == b'zstd':
deltablob = self._cctx.compress(delta)
compression = COMPRESSION_ZSTD
elif self._compengine == b'zlib':
deltablob = zlib.compress(delta)
compression = COMPRESSION_ZLIB
elif self._compengine == b'none':
deltablob = delta
compression = COMPRESSION_NONE
else:
raise error.ProgrammingError(
b'unhandled compression engine: %s' % self._compengine
)
# Don't store compressed data if it isn't practical.
if len(deltablob) >= len(delta):
deltablob = delta
compression = COMPRESSION_NONE
deltaid = insertdelta(self._db, compression, deltahash, deltablob)
rev = len(self)
if p1 == nullid:
p1rev = nullrev
else:
p1rev = self._nodetorev[p1]
if p2 == nullid:
p2rev = nullrev
else:
p2rev = self._nodetorev[p2]
rid = self._db.execute(
'INSERT INTO fileindex ('
' pathid, revnum, node, p1rev, p2rev, linkrev, flags, '
' deltaid, deltabaseid) '
' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
(
self._pathid,
rev,
node,
p1rev,
p2rev,
linkrev,
flags,
deltaid,
baseid,
),
).lastrowid
entry = revisionentry(
rid=rid,
rev=rev,
node=node,
p1rev=p1rev,
p2rev=p2rev,
p1node=p1,
p2node=p2,
linkrev=linkrev,
flags=flags,
)
self._nodetorev[node] = rev
self._revtonode[rev] = node
self._revisions[node] = entry
return node
def getlocalkey(file, id):
pathhash = hex(hashutil.sha1(file).digest())
return os.path.join(pathhash, id)
def getcachekey(reponame, file, id):
pathhash = hex(hashutil.sha1(file).digest())
return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
def hashdiffopts(diffopts):
diffoptstr = stringutil.pprint(
sorted((k, getattr(diffopts, k)) for k in mdiff.diffopts.defaults)
)
return hex(hashutil.sha1(diffoptstr).digest())[:6]
def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles,
matcher, size, lfiletohash):
# Convert src parents to dst parents
parents = _convertparents(ctx, revmap)
# Generate list of changed files
files = _getchangedfiles(ctx, parents)
dstfiles = []
for f in files:
if f not in lfiles and f not in normalfiles:
islfile = _islfile(f, ctx, matcher, size)
# If this file was renamed or copied then copy
# the largefile-ness of its predecessor
if f in ctx.manifest():
fctx = ctx.filectx(f)
renamed = fctx.copysource()
if renamed is None:
# the code below assumes renamed to be a boolean or a list
# and won't quite work with the value None
renamed = False
renamedlfile = renamed and renamed in lfiles
islfile |= renamedlfile
if b'l' in fctx.flags():
if renamedlfile:
raise error.Abort(
_(b'renamed/copied largefile %s becomes symlink') %
f)
islfile = False
if islfile:
lfiles.add(f)
else:
normalfiles.add(f)
if f in lfiles:
fstandin = lfutil.standin(f)
dstfiles.append(fstandin)
# largefile in manifest if it has not been removed/renamed
if f in ctx.manifest():
fctx = ctx.filectx(f)
if b'l' in fctx.flags():
renamed = fctx.copysource()
if renamed and renamed in lfiles:
raise error.Abort(
_(b'largefile %s becomes symlink') % f)
# largefile was modified, update standins
m = hashutil.sha1(b'')
m.update(ctx[f].data())
hash = hex(m.digest())
if f not in lfiletohash or lfiletohash[f] != hash:
rdst.wwrite(f, ctx[f].data(), ctx[f].flags())
executable = b'x' in ctx[f].flags()
lfutil.writestandin(rdst, fstandin, hash, executable)
lfiletohash[f] = hash
else:
# normal file
dstfiles.append(f)
def getfilectx(repo, memctx, f):
srcfname = lfutil.splitstandin(f)
if srcfname is not None:
# if the file isn't in the manifest then it was removed
# or renamed, return None to indicate this
try:
fctx = ctx.filectx(srcfname)
except error.LookupError:
return None
renamed = fctx.copysource()
if renamed:
# standin is always a largefile because largefile-ness
# doesn't change after rename or copy
renamed = lfutil.standin(renamed)
return context.memfilectx(
repo,
memctx,
f,
lfiletohash[srcfname] + b'\n',
b'l' in fctx.flags(),
b'x' in fctx.flags(),
renamed,
)
else:
return _getnormalcontext(repo, ctx, f, revmap)
# Commit
_commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap)
def _findsection(self, name):
params = self.params
namehash = hashutil.sha1(name).digest()
fanoutkey = struct.unpack(
params.fanoutstruct, namehash[: params.fanoutprefix]
)[0]
fanout = self._fanouttable
start = fanout[fanoutkey] + params.indexstart
indexend = self._indexend
for i in pycompat.xrange(fanoutkey + 1, params.fanoutcount):
end = fanout[i] + params.indexstart
if end != start:
break
else:
end = indexend
entry = self._bisect(namehash, start, end, self.INDEXENTRYLENGTH)
if not entry:
raise KeyError(name)
rawentry = struct.unpack(self.INDEXFORMAT, entry)
x, offset, size, nodeindexoffset, nodeindexsize = rawentry
rawnamelen = self._index[
nodeindexoffset : nodeindexoffset + constants.FILENAMESIZE
]
actualnamelen = struct.unpack(b'!H', rawnamelen)[0]
nodeindexoffset += constants.FILENAMESIZE
actualname = self._index[
nodeindexoffset : nodeindexoffset + actualnamelen
]
if actualname != name:
raise KeyError(
b"found file name %s when looking for %s" % (actualname, name)
)
nodeindexoffset += actualnamelen
filenamelength = struct.unpack(
b'!H', self._data[offset : offset + constants.FILENAMESIZE]
)[0]
offset += constants.FILENAMESIZE
actualname = self._data[offset : offset + filenamelength]
offset += filenamelength
if name != actualname:
raise KeyError(
b"found file name %s when looking for %s" % (actualname, name)
)
# Skip entry list size
offset += ENTRYCOUNTSIZE
nodelistoffset = offset
nodelistsize = (
size - constants.FILENAMESIZE - filenamelength - ENTRYCOUNTSIZE
)
return (
name,
nodelistoffset,
nodelistsize,
nodeindexoffset,
nodeindexsize,
)