def _check_drec(self, drec):
# drec has members oid, tid, data, data_txn
tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
ref2name = self._ref2name
ref2name_get = ref2name.get
records_map_get = self._records_map.get
if pick:
oid_in_oids = oid in self.oids
for ref, klass in get_refs(pick):
if ref in self.oids:
oidclass = ref2name_get(oid, None)
if oidclass is None:
ref2name[oid] = oidclass = get_class(pick)
self._msg(ref, tid, "referenced by", oid_repr(oid),
oidclass, "at", pos)
if oid_in_oids:
if klass is None:
klass = ref2name_get(ref, None)
if klass is None:
r = records_map_get(ref, None)
# For save memory we only save references
# seen in one transaction with interesting
# objects changes. So in some circumstances
# we may still got "<unknown>" class name.
if r is None:
klass = "<unknown>"
else:
ref2name[ref] = klass = get_class(r.data)
elif isinstance(klass, tuple):
ref2name[ref] = klass = "%s.%s" % klass
self._msg(oid, tid, "references", oid_repr(ref), klass,
"at", pos)
def _check_drec(self, drec):
# drec has members oid, tid, data, data_txn
tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
ref2name = self._ref2name
ref2name_get = ref2name.get
records_map_get = self._records_map.get
if pick:
oid_in_oids = oid in self.oids
for ref, klass in get_refs(pick):
if ref in self.oids:
oidclass = ref2name_get(oid, None)
if oidclass is None:
ref2name[oid] = oidclass = get_class(pick)
self._msg(ref, tid, "referenced by", oid_repr(oid),
oidclass, "at", pos)
if oid_in_oids:
if klass is None:
klass = ref2name_get(ref, None)
if klass is None:
r = records_map_get(ref, None)
# For save memory we only save references
# seen in one transaction with interesting
# objects changes. So in some circumstances
# we may still got "<unknown>" class name.
if r is None:
klass = "<unknown>"
else:
ref2name[ref] = klass = get_class(r.data)
elif isinstance(klass, tuple):
ref2name[ref] = klass = "%s.%s" % klass
self._msg(oid, tid, "references", oid_repr(ref), klass,
"at", pos)
def main(path=None):
verbose = 0
if path is None:
import sys
import getopt
opts, args = getopt.getopt(sys.argv[1:], "v")
for k, v in opts:
if k == "-v":
verbose += 1
path, = args
fs = FileStorage(path, read_only=1)
# Set of oids in the index that failed to load due to POSKeyError.
# This is what happens if undo is applied to the transaction creating
# the object (the oid is still in the index, but its current data
# record has a backpointer of 0, and POSKeyError is raised then
# because of that backpointer).
undone = {}
# Set of oids that were present in the index but failed to load.
# This does not include oids in undone.
noload = {}
for oid in fs._index.keys():
try:
data, serial = load_current(fs, oid)
except (KeyboardInterrupt, SystemExit):
raise
except POSKeyError:
undone[oid] = 1
except:
if verbose:
traceback.print_exc()
noload[oid] = 1
inactive = noload.copy()
inactive.update(undone)
for oid in fs._index.keys():
if oid in inactive:
continue
data, serial = load_current(fs, oid)
refs = get_refs(data)
missing = [] # contains 3-tuples of oid, klass-metadata, reason
for ref, klass in refs:
if klass is None:
klass = '<unknown>'
if ref not in fs._index:
missing.append((ref, klass, "missing"))
if ref in noload:
missing.append((ref, klass, "failed to load"))
if ref in undone:
missing.append((ref, klass, "object creation was undone"))
if missing:
report(oid, data, serial, missing)
def main(path):
fs = FileStorage(path, read_only=1)
# Set of oids in the index that failed to load due to POSKeyError.
# This is what happens if undo is applied to the transaction creating
# the object (the oid is still in the index, but its current data
# record has a backpointer of 0, and POSKeyError is raised then
# because of that backpointer).
undone = {}
# Set of oids that were present in the index but failed to load.
# This does not include oids in undone.
noload = {}
for oid in fs._index.keys():
try:
data, serial = fs.load(oid, "")
except (KeyboardInterrupt, SystemExit):
raise
except POSKeyError:
undone[oid] = 1
except:
if VERBOSE:
traceback.print_exc()
noload[oid] = 1
inactive = noload.copy()
inactive.update(undone)
for oid in fs._index.keys():
if oid in inactive:
continue
data, serial = fs.load(oid, "")
refs = get_refs(data)
missing = [] # contains 3-tuples of oid, klass-metadata, reason
for info in refs:
ref, klass = info
if klass is None:
# failed to unpack
ref = info
klass = '<unknown>'
if ref not in fs._index:
missing.append((ref, klass, "missing"))
if ref in noload:
missing.append((ref, klass, "failed to load"))
if ref in undone:
missing.append((ref, klass, "object creation was undone"))
if missing:
report(oid, data, serial, missing)
def main(path=None):
verbose = 0
if path is None:
import sys
import getopt
opts, args = getopt.getopt(sys.argv[1:], "v")
for k, v in opts:
if k == "-v":
verbose += 1
path, = args
fs = FileStorage(path, read_only=1)
# Set of oids in the index that failed to load due to POSKeyError.
# This is what happens if undo is applied to the transaction creating
# the object (the oid is still in the index, but its current data
# record has a backpointer of 0, and POSKeyError is raised then
# because of that backpointer).
undone = {}
# Set of oids that were present in the index but failed to load.
# This does not include oids in undone.
noload = {}
# build {pos -> oid} index that is reverse to {oid -> pos} fs._index
# we'll need this to iterate objects in order of ascending file position to
# optimize disk IO.
pos2oid = QQBTree() # pos -> u64(oid)
for oid, pos in fs._index.iteritems():
pos2oid[pos] = u64(oid)
# pass 1: load all objects listed in the index and remember those objects
# that are deleted or load with an error. Iterate objects in order of
# ascending file position to optimize disk IO.
for oid64 in pos2oid.itervalues():
oid = p64(oid64)
try:
data, serial = load_current(fs, oid)
except (KeyboardInterrupt, SystemExit):
raise
except POSKeyError:
undone[oid] = 1
except:
if verbose:
traceback.print_exc()
noload[oid] = 1
# pass 2: go through all objects again and verify that their references do
# not point to problematic object set. Iterate objects in order of ascending
# file position to optimize disk IO.
inactive = noload.copy()
inactive.update(undone)
for oid64 in pos2oid.itervalues():
oid = p64(oid64)
if oid in inactive:
continue
data, serial = load_current(fs, oid)
refs = get_refs(data)
missing = [] # contains 3-tuples of oid, klass-metadata, reason
for ref, klass in refs:
if klass is None:
klass = '<unknown>'
if ref not in fs._index:
missing.append((ref, klass, "missing"))
if ref in noload:
missing.append((ref, klass, "failed to load"))
if ref in undone:
missing.append((ref, klass, "object creation was undone"))
if missing:
report(oid, data, serial, missing)