def test_KnownConstants(self):
self.assertEqual(b"\000\000\000\000\000\000\000\001", p64(1))
self.assertEqual(b"\000\000\000\001\000\000\000\000", p64(1 << 32))
self.assertEqual(u64(b"\000\000\000\000\000\000\000\001"), 1)
self.assertEqual(U64(b"\000\000\000\000\000\000\000\001"), 1)
self.assertEqual(u64(b"\000\000\000\001\000\000\000\000"), 1 << 32)
self.assertEqual(U64(b"\000\000\000\001\000\000\000\000"), 1 << 32)
def checkKnownConstants(self):
self.assertEquals("\000\000\000\000\000\000\000\001", p64(1))
self.assertEquals("\000\000\000\001\000\000\000\000", p64(1L << 32))
self.assertEquals(u64("\000\000\000\000\000\000\000\001"), 1)
self.assertEquals(U64("\000\000\000\000\000\000\000\001"), 1)
self.assertEquals(u64("\000\000\000\001\000\000\000\000"), 1L << 32)
self.assertEquals(U64("\000\000\000\001\000\000\000\000"), 1L << 32)
def checkCorruptionInPack(self):
# This sets up a corrupt .fs file, with a redundant transaction
# length mismatch. The implementation of pack in many releases of
# ZODB blew up if the .fs file had such damage: it detected the
# damage, but the code to raise CorruptedError referenced an undefined
# global.
import time
from ZODB.utils import U64, p64
from ZODB.FileStorage.format import CorruptedError
from ZODB.serialize import referencesf
db = DB(self._storage)
conn = db.open()
conn.root()['xyz'] = 1
transaction.commit()
# Ensure it's all on disk.
db.close()
self._storage.close()
# Reopen before damaging.
self.open()
# Open .fs directly, and damage content.
with open('FileStorageTests.fs', 'r+b') as f:
f.seek(0, 2)
pos2 = f.tell() - 8
f.seek(pos2)
tlen2 = U64(f.read(8)) # length-8 of the last transaction
pos1 = pos2 - tlen2 + 8 # skip over the tid at the start
f.seek(pos1)
tlen1 = U64(f.read(8)) # should be redundant length-8
self.assertEqual(tlen1, tlen2) # verify that it is redundant
# Now damage the second copy.
f.seek(pos2)
f.write(p64(tlen2 - 1))
# Try to pack. This used to yield
# NameError: global name 's' is not defined
try:
self._storage.pack(time.time(), referencesf)
except CorruptedError as detail:
self.assertTrue("redundant transaction length does not match "
"initial transaction length" in str(detail))
else:
self.fail("expected CorruptedError")
def test_LongToStringToLong(self):
for num in self.all:
s = p64(num)
n = U64(s)
self.assertEqual(num, n, "U64() failed")
n2 = u64(s)
self.assertEqual(num, n2, "u64() failed")
def checkExtendedIteration(self):
# Store a bunch of revisions of a single object
self._oid = oid = self._storage.new_oid()
revid1 = self._dostore(oid, data=MinPO(11))
revid2 = self._dostore(oid, revid=revid1, data=MinPO(12))
revid3 = self._dostore(oid, revid=revid2, data=MinPO(13))
revid4 = self._dostore(oid, revid=revid3, data=MinPO(14))
# Note that the end points are included
# Iterate over all of the transactions with explicit start/stop
txniter = self._storage.iterator(revid1, revid4)
self.iter_verify(txniter, [revid1, revid2, revid3, revid4], 11)
# Iterate over some of the transactions with explicit start
txniter = self._storage.iterator(revid3)
self.iter_verify(txniter, [revid3, revid4], 13)
# Iterate over some of the transactions with explicit stop
txniter = self._storage.iterator(None, revid2)
self.iter_verify(txniter, [revid1, revid2], 11)
# Iterate over some of the transactions with explicit start+stop
txniter = self._storage.iterator(revid2, revid3)
self.iter_verify(txniter, [revid2, revid3], 12)
# Specify an upper bound somewhere in between values
revid3a = p64((U64(revid3) + U64(revid4)) / 2)
txniter = self._storage.iterator(revid2, revid3a)
self.iter_verify(txniter, [revid2, revid3], 12)
# Specify a lower bound somewhere in between values.
# revid2 == revid1+1 is very likely on Windows. Adding 1 before
# dividing ensures that "the midpoint" we compute is strictly larger
# than revid1.
revid1a = p64((U64(revid1) + 1 + U64(revid2)) / 2)
assert revid1 < revid1a
txniter = self._storage.iterator(revid1a, revid3a)
self.iter_verify(txniter, [revid2, revid3], 12)
# Specify an empty range
txniter = self._storage.iterator(revid3, revid2)
self.iter_verify(txniter, [], 13)
# Specify a singleton range
txniter = self._storage.iterator(revid3, revid3)
self.iter_verify(txniter, [revid3], 13)
def main(path):
fs = FileStorage(path, read_only=1)
if PACK:
fs.pack()
db = ZODB.DB(fs)
rt = db.open().root()
paths = find_paths(rt, 3)
def total_size(oid):
cache = {}
cache_size = 1000
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, '')
size = len(data)
for suboid in referencesf(data):
if seen.has_key(suboid):
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
return _total_size(oid, {})
keys = fs._index.keys()
keys.sort()
keys.reverse()
if not VERBOSE:
# If not running verbosely, don't print an entry for an object
# unless it has an entry in paths.
keys = filter(paths.has_key, keys)
fmt = "%8s %5d %8d %s %s.%s"
for oid in keys:
data, serialno = fs.load(oid, '')
mod, klass = get_pickle_metadata(data)
refs = referencesf(data)
path = paths.get(oid, '-')
print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
def run(path, v=0):
fs = FileStorage(path, read_only=1)
# break into the file implementation
if hasattr(fs._index, 'iterkeys'):
iter = six.iterkeys(fs._index)
else:
iter = fs._index.keys()
totals = {}
for oid in iter:
data, serialno = fs.load(oid, '')
mod, klass = get_pickle_metadata(data)
key = "%s.%s" % (mod, klass)
bytes, count = totals.get(key, (0, 0))
bytes += len(data)
count += 1
totals[key] = bytes, count
if v:
print("%8s %5d %s" % (U64(oid), len(data), key))
L = totals.items()
L.sort(lambda a, b: cmp(a[1], b[1]))
L.reverse()
print("Totals per object class:")
for key, (bytes, count) in L:
print("%8d %8d %s" % (count, bytes, key))
def oid_repr(oid):
if isinstance(oid, StringType) and len(oid) == 8:
return '%16x' % U64(oid)
else:
return repr(oid)
