def main(path):
fs = FileStorage(path, read_only=1)
if PACK:
fs.pack()
db = ZODB.DB(fs)
rt = db.open().root()
paths = find_paths(rt, 3)
def total_size(oid):
cache = {}
cache_size = 1000
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, '')
size = len(data)
for suboid in referencesf(data):
if seen.has_key(suboid):
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
return _total_size(oid, {})
keys = fs._index.keys()
keys.sort()
keys.reverse()
if not VERBOSE:
# If not running verbosely, don't print an entry for an object
# unless it has an entry in paths.
keys = filter(paths.has_key, keys)
fmt = "%8s %5d %8d %s %s.%s"
for oid in keys:
data, serialno = fs.load(oid, '')
mod, klass = get_pickle_metadata(data)
refs = referencesf(data)
path = paths.get(oid, '-')
print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
def main(path):
fs = FileStorage(path, read_only=1)
if PACK:
fs.pack()
db = ZODB.DB(fs)
rt = db.open().root()
paths = find_paths(rt, 3)
def total_size(oid):
cache = {}
cache_size = 1000
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, "")
size = len(data)
for suboid in referencesf(data):
if seen.has_key(suboid):
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
return _total_size(oid, {})
keys = fs._index.keys()
keys.sort()
keys.reverse()
if not VERBOSE:
# If not running verbosely, don't print an entry for an object
# unless it has an entry in paths.
keys = filter(paths.has_key, keys)
fmt = "%8s %5d %8d %s %s.%s"
for oid in keys:
data, serialno = fs.load(oid, "")
mod, klass = get_pickle_metadata(data)
refs = referencesf(data)
path = paths.get(oid, "-")
print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
class ZODBClassifier(object):
ClassifierClass = _PersistentClassifier
def __init__(self, db_name, mode="c"):
self.db_filename = db_name
self.db_name = os.path.basename(db_name)
self.closed = True
self.mode = mode
self.load()
def __getattr__(self, att):
if hasattr(self, "classifier") and hasattr(self.classifier, att):
return getattr(self.classifier, att)
raise AttributeError("ZODBClassifier object has no attribute '%s'" % (att,))
def __setattr__(self, att, value):
if att in ("nham", "nspam") and hasattr(self, "classifier"):
setattr(self.classifier, att, value)
else:
object.__setattr__(self, att, value)
def create_storage(self):
from ZODB.FileStorage import FileStorage
try:
self.storage = FileStorage(self.db_filename, read_only=self.mode == "r")
except IOError:
print(("Could not create FileStorage from", self.db_filename), file=sys.stderr)
raise
def load(self):
"""Load state from database"""
import ZODB
if options["globals", "verbose"]:
print("Loading state from %s (%s) database" % (self.db_filename, self.db_name), file=sys.stderr)
if not self.closed:
self.close()
self.create_storage()
self.DB = ZODB.DB(self.storage, cache_size=10000)
self.conn = self.DB.open()
root = self.conn.root()
self.classifier = root.get(self.db_name)
if self.classifier is None:
if options["globals", "verbose"]:
print(self.db_name, "is a new ZODB", file=sys.stderr)
self.classifier = root[self.db_name] = self.ClassifierClass()
else:
if options["globals", "verbose"]:
print(
"%s is an existing ZODB, with %d " "ham and %d spam" % (self.db_name, self.nham, self.nspam),
file=sys.stderr,
)
self.closed = False
def store(self):
"""Place state into persistent store"""
try:
import ZODB.Transaction
except ImportError:
import transaction
commit = transaction.commit
abort = transaction.abort
else:
commit = ZODB.Transaction.get_transaction().commit
abort = ZODB.Transaction.get_transaction().abort
from ZODB.POSException import ConflictError
try:
from ZODB.POSException import TransactionFailedError
except:
from ZODB.POSException import TransactionError as TransactionFailedError
from ZODB.POSException import ReadOnlyError
assert not self.closed, "Can't store a closed database"
if options["globals", "verbose"]:
print("Persisting", self.db_name, "state in database", file=sys.stderr)
try:
commit()
except ConflictError:
if options["globals", "verbose"]:
print("Conflict on commit", self.db_name, file=sys.stderr)
abort()
except TransactionFailedError:
print("Storing failed. Need to restart.", self.db_name, file=sys.stderr)
abort()
except ReadOnlyError:
print("Can't store transaction to read-only db.", file=sys.stderr)
abort()
def close(self, pack=True, retain_backup=True):
if self.mode != "r":
self.store()
if pack and self.mode != "r":
self.pack(time.time() - 60 * 60 * 24, retain_backup)
self.DB.close()
self.storage.close()
delattr(self, "classifier")
self.closed = True
if options["globals", "verbose"]:
print("Closed", self.db_name, "database", file=sys.stderr)
def pack(self, t, retain_backup=True):
"""Like FileStorage pack(), but optionally remove the .old
backup file that is created. Often for our purposes we do
not care about being able to recover from this. Also
ignore the referencesf parameter, which appears to not do
anything."""
if hasattr(self.storage, "pack"):
self.storage.pack(t, None)
if not retain_backup:
old_name = self.db_filename + ".old"
if os.path.exists(old_name):
os.remove(old_name)
class ZODBClassifier(object):
# Allow subclasses to override classifier class.
ClassifierClass = _PersistentClassifier
def __init__(self, db_name, mode='c'):
self.db_filename = db_name
self.db_name = os.path.basename(db_name)
self.closed = True
self.mode = mode
self.load()
def __getattr__(self, att):
# We pretend that we are a classifier subclass.
if hasattr(self, "classifier") and hasattr(self.classifier, att):
return getattr(self.classifier, att)
raise AttributeError("ZODBClassifier object has no attribute '%s'"
% (att,))
def __setattr__(self, att, value):
# For some attributes, we change the classifier instead.
if att in ("nham", "nspam") and hasattr(self, "classifier"):
setattr(self.classifier, att, value)
else:
object.__setattr__(self, att, value)
def create_storage(self):
from ZODB.FileStorage import FileStorage
try:
self.storage = FileStorage(self.db_filename,
read_only=self.mode=='r')
except IOError:
print(("Could not create FileStorage from",
self.db_filename), file=sys.stderr)
raise
def load(self):
'''Load state from database'''
import ZODB
if options["globals", "verbose"]:
print("Loading state from %s (%s) database" % \
(self.db_filename, self.db_name), file=sys.stderr)
# If we are not closed, then we need to close first before we
# reload.
if not self.closed:
self.close()
self.create_storage()
self.DB = ZODB.DB(self.storage, cache_size=10000)
self.conn = self.DB.open()
root = self.conn.root()
self.classifier = root.get(self.db_name)
if self.classifier is None:
# There is no classifier, so create one.
if options["globals", "verbose"]:
print(self.db_name, 'is a new ZODB', file=sys.stderr)
self.classifier = root[self.db_name] = self.ClassifierClass()
else:
if options["globals", "verbose"]:
print('%s is an existing ZODB, with %d ' \
'ham and %d spam' % (self.db_name, self.nham,
self.nspam), file=sys.stderr)
self.closed = False
def store(self):
'''Place state into persistent store'''
try:
import ZODB.Transaction
except ImportError:
import transaction
commit = transaction.commit
abort = transaction.abort
else:
commit = ZODB.Transaction.get_transaction().commit
abort = ZODB.Transaction.get_transaction().abort
from ZODB.POSException import ConflictError
try:
from ZODB.POSException import TransactionFailedError
except:
from ZODB.POSException import TransactionError as TransactionFailedError
from ZODB.POSException import ReadOnlyError
assert not self.closed, "Can't store a closed database"
if options["globals", "verbose"]:
print('Persisting', self.db_name, 'state in database', file=sys.stderr)
try:
commit()
except ConflictError:
# We'll save it next time, or on close. It'll be lost if we
# hard-crash, but that's unlikely, and not a particularly big
# deal.
if options["globals", "verbose"]:
print("Conflict on commit", self.db_name, file=sys.stderr)
abort()
except TransactionFailedError:
# Saving isn't working. Try to abort, but chances are that
# restarting is needed.
print("Storing failed. Need to restart.", \
self.db_name, file=sys.stderr)
abort()
except ReadOnlyError:
print("Can't store transaction to read-only db.", file=sys.stderr)
abort()
def close(self, pack=True, retain_backup=True):
# Ensure that the db is saved before closing. Alternatively, we
# could abort any waiting transaction. We need to do *something*
# with it, though, or it will be still around after the db is
# closed and cause problems. For now, saving seems to make sense
# (and we can always add abort methods if they are ever needed).
if self.mode != 'r':
self.store()
# We don't make any use of the 'undo' capabilities of the
# FileStorage at the moment, so might as well pack the database
# each time it is closed, to save as much disk space as possible.
# Pack it up to where it was 'yesterday'.
if pack and self.mode != 'r':
self.pack(time.time()-60*60*24, retain_backup)
# Do the closing.
self.DB.close()
self.storage.close()
# Ensure that we cannot continue to use this classifier.
delattr(self, "classifier")
self.closed = True
if options["globals", "verbose"]:
print('Closed', self.db_name, 'database', file=sys.stderr)
def pack(self, t, retain_backup=True):
"""Like FileStorage pack(), but optionally remove the .old
backup file that is created. Often for our purposes we do
not care about being able to recover from this. Also
ignore the referencesf parameter, which appears to not do
anything."""
if hasattr(self.storage, "pack"):
self.storage.pack(t, None)
if not retain_backup:
old_name = self.db_filename + ".old"
if os.path.exists(old_name):
os.remove(old_name)
