def pack(self, t, referencesf): self._lock_acquire() try: rindex={} referenced=rindex.has_key rootl=['\0\0\0\0\0\0\0\0'] # mark referenced objects while rootl: oid=rootl.pop() if referenced(oid): continue p = self._opickle[oid] referencesf(p, rootl) rindex[oid] = None # sweep unreferenced objects for oid in self._index.keys(): if not referenced(oid): self._takeOutGarbage(oid) finally: self._lock_release()
def generate_durus_object_records(): sio = cStringIO.StringIO() zodb_storage = ZODBFileStorage(zodb_file_name) n = 0 for oid in zodb_storage._index.keys(): n += 1 if n % 10000 == 0: sys.stdout.write('.') sys.stdout.flush() p, serial = zodb_storage.load(oid, '') refs = referencesf(p) # unwrap extra tuple from class meta data sio.seek(0) sio.write(p) sio.truncate() sio.seek(0) def get_class(module_class): module, klass = module_class if module not in sys.modules: __import__(module) return getattr(sys.modules[module], klass) class PersistentRef: def __init__(self, v): oid, module_class = v self.oid_klass = (oid, get_class(module_class)) unpickler = cPickle.Unpickler(sio) unpickler.persistent_load = lambda v: PersistentRef(v) class_meta = unpickler.load() class_meta, extra = class_meta assert extra is None object_state = unpickler.load() if type(object_state) == dict and '_container' in object_state: assert 'data' not in object_state object_state['data'] = object_state['_container'] del object_state['_container'] sio.seek(0) sio.truncate() cPickle.dump(get_class(class_meta), sio, 2) pickler = cPickle.Pickler(sio, 2) def persistent_id(v): if isinstance(v, PersistentRef): return v.oid_klass return None pickler.persistent_id = persistent_id pickler.dump(object_state) record = pack_record(oid, sio.getvalue(), ''.join(refs)) yield record print print n, 'objects written'
def _total_size(oid, seen): v = cache.get(oid) if v is not None: return v data, serialno = fs.load(oid, '') size = len(data) for suboid in referencesf(data): if seen.has_key(suboid): continue seen[suboid] = 1 size += _total_size(suboid, seen) cache[oid] = size if len(cache) == cache_size: cache.popitem() return size
def _total_size(oid, seen): v = cache.get(oid) if v is not None: return v data, serialno = fs.load(oid, '') size = len(data) for suboid in referencesf(data): if suboid in seen: continue seen[suboid] = 1 size += _total_size(suboid, seen) cache[oid] = size if len(cache) == cache_size: cache.popitem() return size
def main(path): fs = FileStorage(path, read_only=1) if PACK: fs.pack() db = ZODB.DB(fs) rt = db.open().root() paths = find_paths(rt, 3) def total_size(oid): cache = {} cache_size = 1000 def _total_size(oid, seen): v = cache.get(oid) if v is not None: return v data, serialno = fs.load(oid, '') size = len(data) for suboid in referencesf(data): if seen.has_key(suboid): continue seen[suboid] = 1 size += _total_size(suboid, seen) cache[oid] = size if len(cache) == cache_size: cache.popitem() return size return _total_size(oid, {}) keys = fs._index.keys() keys.sort() keys.reverse() if not VERBOSE: # If not running verbosely, don't print an entry for an object # unless it has an entry in paths. keys = filter(paths.has_key, keys) fmt = "%8s %5d %8d %s %s.%s" for oid in keys: data, serialno = fs.load(oid, '') mod, klass = get_pickle_metadata(data) refs = referencesf(data) path = paths.get(oid, '-') print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
def main(path): fs = FileStorage(path, read_only=1) if PACK: fs.pack() db = ZODB.DB(fs) rt = db.open().root() paths = find_paths(rt, 3) def total_size(oid): cache = {} cache_size = 1000 def _total_size(oid, seen): v = cache.get(oid) if v is not None: return v data, serialno = fs.load(oid, "") size = len(data) for suboid in referencesf(data): if seen.has_key(suboid): continue seen[suboid] = 1 size += _total_size(suboid, seen) cache[oid] = size if len(cache) == cache_size: cache.popitem() return size return _total_size(oid, {}) keys = fs._index.keys() keys.sort() keys.reverse() if not VERBOSE: # If not running verbosely, don't print an entry for an object # unless it has an entry in paths. keys = filter(paths.has_key, keys) fmt = "%8s %5d %8d %s %s.%s" for oid in keys: data, serialno = fs.load(oid, "") mod, klass = get_pickle_metadata(data) refs = referencesf(data) path = paths.get(oid, "-") print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
def _finish(self, tid, u, d, e): zeros={} referenceCount=self._referenceCount referenceCount_get=referenceCount.get oreferences=self._oreferences serial=self._serial index=self._index opickle=self._opickle # iterate over all the objects touched by/created within this # transaction for entry in self._tmp: oid, data = entry[:] referencesl=[] referencesf(data, referencesl) references={} for roid in referencesl: references[roid]=1 referenced=references.has_key # Create a reference count for this object if one # doesn't already exist if referenceCount_get(oid) is None: referenceCount[oid] = 0 #zeros[oid]=1 # update references that are already associated with this # object roids = oreferences.get(oid, []) for roid in roids: if referenced(roid): # still referenced, so no need to update # remove it from the references dict so it doesn't # get "added" in the next clause del references[roid] else: # Delete the stored ref, since we no longer # have it oreferences[oid].remove(roid) # decrement refcnt: rc = referenceCount_get(roid, 1) rc=rc-1 if rc < 0: # This should never happen raise ReferenceCountError, ( "%s (Oid %s had refcount %s)" % (ReferenceCountError.__doc__,`roid`,rc) ) referenceCount[roid] = rc if rc==0: zeros[roid]=1 # Create a reference list for this object if one # doesn't already exist if oreferences.get(oid) is None: oreferences[oid] = [] # Now add any references that weren't already stored for roid in references.keys(): oreferences[oid].append(roid) # Create/update refcnt rc=referenceCount_get(roid, 0) if rc==0 and zeros.get(roid) is not None: del zeros[roid] referenceCount[roid] = rc+1 index[oid] = serial opickle[oid] = data if zeros: for oid in zeros.keys(): if oid == '\0\0\0\0\0\0\0\0': continue self._takeOutGarbage(oid) self._tmp = []