def pack(self, t, referencesf):
self._lock_acquire()
try:
rindex={}
referenced=rindex.has_key
rootl=['\0\0\0\0\0\0\0\0']
# mark referenced objects
while rootl:
oid=rootl.pop()
if referenced(oid): continue
p = self._opickle[oid]
referencesf(p, rootl)
rindex[oid] = None
# sweep unreferenced objects
for oid in self._index.keys():
if not referenced(oid):
self._takeOutGarbage(oid)
finally:
self._lock_release()
def generate_durus_object_records():
sio = cStringIO.StringIO()
zodb_storage = ZODBFileStorage(zodb_file_name)
n = 0
for oid in zodb_storage._index.keys():
n += 1
if n % 10000 == 0:
sys.stdout.write('.')
sys.stdout.flush()
p, serial = zodb_storage.load(oid, '')
refs = referencesf(p)
# unwrap extra tuple from class meta data
sio.seek(0)
sio.write(p)
sio.truncate()
sio.seek(0)
def get_class(module_class):
module, klass = module_class
if module not in sys.modules:
__import__(module)
return getattr(sys.modules[module], klass)
class PersistentRef:
def __init__(self, v):
oid, module_class = v
self.oid_klass = (oid, get_class(module_class))
unpickler = cPickle.Unpickler(sio)
unpickler.persistent_load = lambda v: PersistentRef(v)
class_meta = unpickler.load()
class_meta, extra = class_meta
assert extra is None
object_state = unpickler.load()
if type(object_state) == dict and '_container' in object_state:
assert 'data' not in object_state
object_state['data'] = object_state['_container']
del object_state['_container']
sio.seek(0)
sio.truncate()
cPickle.dump(get_class(class_meta), sio, 2)
pickler = cPickle.Pickler(sio, 2)
def persistent_id(v):
if isinstance(v, PersistentRef):
return v.oid_klass
return None
pickler.persistent_id = persistent_id
pickler.dump(object_state)
record = pack_record(oid, sio.getvalue(), ''.join(refs))
yield record
print
print n, 'objects written'
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, '')
size = len(data)
for suboid in referencesf(data):
if seen.has_key(suboid):
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, '')
size = len(data)
for suboid in referencesf(data):
if suboid in seen:
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
def main(path):
fs = FileStorage(path, read_only=1)
if PACK:
fs.pack()
db = ZODB.DB(fs)
rt = db.open().root()
paths = find_paths(rt, 3)
def total_size(oid):
cache = {}
cache_size = 1000
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, '')
size = len(data)
for suboid in referencesf(data):
if seen.has_key(suboid):
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
return _total_size(oid, {})
keys = fs._index.keys()
keys.sort()
keys.reverse()
if not VERBOSE:
# If not running verbosely, don't print an entry for an object
# unless it has an entry in paths.
keys = filter(paths.has_key, keys)
fmt = "%8s %5d %8d %s %s.%s"
for oid in keys:
data, serialno = fs.load(oid, '')
mod, klass = get_pickle_metadata(data)
refs = referencesf(data)
path = paths.get(oid, '-')
print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
def main(path):
fs = FileStorage(path, read_only=1)
if PACK:
fs.pack()
db = ZODB.DB(fs)
rt = db.open().root()
paths = find_paths(rt, 3)
def total_size(oid):
cache = {}
cache_size = 1000
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, "")
size = len(data)
for suboid in referencesf(data):
if seen.has_key(suboid):
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
return _total_size(oid, {})
keys = fs._index.keys()
keys.sort()
keys.reverse()
if not VERBOSE:
# If not running verbosely, don't print an entry for an object
# unless it has an entry in paths.
keys = filter(paths.has_key, keys)
fmt = "%8s %5d %8d %s %s.%s"
for oid in keys:
data, serialno = fs.load(oid, "")
mod, klass = get_pickle_metadata(data)
refs = referencesf(data)
path = paths.get(oid, "-")
print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
def _finish(self, tid, u, d, e):
zeros={}
referenceCount=self._referenceCount
referenceCount_get=referenceCount.get
oreferences=self._oreferences
serial=self._serial
index=self._index
opickle=self._opickle
# iterate over all the objects touched by/created within this
# transaction
for entry in self._tmp:
oid, data = entry[:]
referencesl=[]
referencesf(data, referencesl)
references={}
for roid in referencesl:
references[roid]=1
referenced=references.has_key
# Create a reference count for this object if one
# doesn't already exist
if referenceCount_get(oid) is None:
referenceCount[oid] = 0
#zeros[oid]=1
# update references that are already associated with this
# object
roids = oreferences.get(oid, [])
for roid in roids:
if referenced(roid):
# still referenced, so no need to update
# remove it from the references dict so it doesn't
# get "added" in the next clause
del references[roid]
else:
# Delete the stored ref, since we no longer
# have it
oreferences[oid].remove(roid)
# decrement refcnt:
rc = referenceCount_get(roid, 1)
rc=rc-1
if rc < 0:
# This should never happen
raise ReferenceCountError, (
"%s (Oid %s had refcount %s)" %
(ReferenceCountError.__doc__,`roid`,rc)
)
referenceCount[roid] = rc
if rc==0:
zeros[roid]=1
# Create a reference list for this object if one
# doesn't already exist
if oreferences.get(oid) is None:
oreferences[oid] = []
# Now add any references that weren't already stored
for roid in references.keys():
oreferences[oid].append(roid)
# Create/update refcnt
rc=referenceCount_get(roid, 0)
if rc==0 and zeros.get(roid) is not None:
del zeros[roid]
referenceCount[roid] = rc+1
index[oid] = serial
opickle[oid] = data
if zeros:
for oid in zeros.keys():
if oid == '\0\0\0\0\0\0\0\0': continue
self._takeOutGarbage(oid)
self._tmp = []
