Example #1
0
    def _check_drec(self, drec):
        # drec has members oid, tid, data, data_txn
        tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
        ref2name = self._ref2name
        ref2name_get = ref2name.get
        records_map_get = self._records_map.get
        if pick:
            oid_in_oids = oid in self.oids
            for ref, klass in get_refs(pick):
                if ref in self.oids:
                    oidclass = ref2name_get(oid, None)
                    if oidclass is None:
                        ref2name[oid] = oidclass = get_class(pick)
                    self._msg(ref, tid, "referenced by", oid_repr(oid),
                              oidclass, "at", pos)

                if oid_in_oids:
                    if klass is None:
                        klass = ref2name_get(ref, None)
                        if klass is None:
                            r = records_map_get(ref, None)
                            # For save memory we only save references
                            # seen in one transaction with interesting
                            # objects changes. So in some circumstances
                            # we may still got "<unknown>" class name.
                            if r is None:
                                klass = "<unknown>"
                            else:
                                ref2name[ref] = klass = get_class(r.data)
                    elif isinstance(klass, tuple):
                        ref2name[ref] = klass = "%s.%s" % klass

                    self._msg(oid, tid, "references", oid_repr(ref), klass,
                              "at", pos)
Example #2
0
    def _check_drec(self, drec):
        # drec has members oid, tid, data, data_txn
        tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
        ref2name = self._ref2name
        ref2name_get = ref2name.get
        records_map_get = self._records_map.get
        if pick:
            oid_in_oids = oid in self.oids
            for ref, klass in get_refs(pick):
                if ref in self.oids:
                    oidclass = ref2name_get(oid, None)
                    if oidclass is None:
                        ref2name[oid] = oidclass = get_class(pick)
                    self._msg(ref, tid, "referenced by", oid_repr(oid),
                              oidclass, "at", pos)

                if oid_in_oids:
                    if klass is None:
                        klass = ref2name_get(ref, None)
                        if klass is None:
                            r = records_map_get(ref, None)
                            # For save memory we only save references
                            # seen in one transaction with interesting
                            # objects changes. So in some circumstances
                            # we may still got "<unknown>" class name.
                            if r is None:
                                klass = "<unknown>"
                            else:
                                ref2name[ref] = klass = get_class(r.data)
                    elif isinstance(klass, tuple):
                        ref2name[ref] = klass = "%s.%s" % klass

                    self._msg(oid, tid, "references", oid_repr(ref), klass,
                              "at", pos)
Example #3
0
def main(path=None):
    verbose = 0
    if path is None:
        import sys
        import getopt

        opts, args = getopt.getopt(sys.argv[1:], "v")
        for k, v in opts:
            if k == "-v":
                verbose += 1

        path, = args


    fs = FileStorage(path, read_only=1)

    # Set of oids in the index that failed to load due to POSKeyError.
    # This is what happens if undo is applied to the transaction creating
    # the object (the oid is still in the index, but its current data
    # record has a backpointer of 0, and POSKeyError is raised then
    # because of that backpointer).
    undone = {}

    # Set of oids that were present in the index but failed to load.
    # This does not include oids in undone.
    noload = {}

    for oid in fs._index.keys():
        try:
            data, serial = load_current(fs, oid)
        except (KeyboardInterrupt, SystemExit):
            raise
        except POSKeyError:
            undone[oid] = 1
        except:
            if verbose:
                traceback.print_exc()
            noload[oid] = 1

    inactive = noload.copy()
    inactive.update(undone)
    for oid in fs._index.keys():
        if oid in inactive:
            continue
        data, serial = load_current(fs, oid)
        refs = get_refs(data)
        missing = [] # contains 3-tuples of oid, klass-metadata, reason
        for ref, klass in refs:
            if klass is None:
                klass = '<unknown>'
            if ref not in fs._index:
                missing.append((ref, klass, "missing"))
            if ref in noload:
                missing.append((ref, klass, "failed to load"))
            if ref in undone:
                missing.append((ref, klass, "object creation was undone"))
        if missing:
            report(oid, data, serial, missing)
Example #4
0
def main(path):
    fs = FileStorage(path, read_only=1)

    # Set of oids in the index that failed to load due to POSKeyError.
    # This is what happens if undo is applied to the transaction creating
    # the object (the oid is still in the index, but its current data
    # record has a backpointer of 0, and POSKeyError is raised then
    # because of that backpointer).
    undone = {}

    # Set of oids that were present in the index but failed to load.
    # This does not include oids in undone.
    noload = {}

    for oid in fs._index.keys():
        try:
            data, serial = fs.load(oid, "")
        except (KeyboardInterrupt, SystemExit):
            raise
        except POSKeyError:
            undone[oid] = 1
        except:
            if VERBOSE:
                traceback.print_exc()
            noload[oid] = 1

    inactive = noload.copy()
    inactive.update(undone)
    for oid in fs._index.keys():
        if oid in inactive:
            continue
        data, serial = fs.load(oid, "")
        refs = get_refs(data)
        missing = [] # contains 3-tuples of oid, klass-metadata, reason
        for info in refs:
            ref, klass = info
            if klass is None:
                # failed to unpack
                ref = info
                klass = '<unknown>'
            if ref not in fs._index:
                missing.append((ref, klass, "missing"))
            if ref in noload:
                missing.append((ref, klass, "failed to load"))
            if ref in undone:
                missing.append((ref, klass, "object creation was undone"))
        if missing:
            report(oid, data, serial, missing)
Example #5
0
def main(path=None):
    verbose = 0
    if path is None:
        import sys
        import getopt

        opts, args = getopt.getopt(sys.argv[1:], "v")
        for k, v in opts:
            if k == "-v":
                verbose += 1

        path, = args

    fs = FileStorage(path, read_only=1)

    # Set of oids in the index that failed to load due to POSKeyError.
    # This is what happens if undo is applied to the transaction creating
    # the object (the oid is still in the index, but its current data
    # record has a backpointer of 0, and POSKeyError is raised then
    # because of that backpointer).
    undone = {}

    # Set of oids that were present in the index but failed to load.
    # This does not include oids in undone.
    noload = {}

    # build {pos -> oid} index that is reverse to {oid -> pos} fs._index
    # we'll need this to iterate objects in order of ascending file position to
    # optimize disk IO.
    pos2oid = QQBTree()  # pos -> u64(oid)
    for oid, pos in fs._index.iteritems():
        pos2oid[pos] = u64(oid)

    # pass 1: load all objects listed in the index and remember those objects
    # that are deleted or load with an error. Iterate objects in order of
    # ascending file position to optimize disk IO.
    for oid64 in pos2oid.itervalues():
        oid = p64(oid64)
        try:
            data, serial = load_current(fs, oid)
        except (KeyboardInterrupt, SystemExit):
            raise
        except POSKeyError:
            undone[oid] = 1
        except:
            if verbose:
                traceback.print_exc()
            noload[oid] = 1

    # pass 2: go through all objects again and verify that their references do
    # not point to problematic object set. Iterate objects in order of ascending
    # file position to optimize disk IO.
    inactive = noload.copy()
    inactive.update(undone)
    for oid64 in pos2oid.itervalues():
        oid = p64(oid64)
        if oid in inactive:
            continue
        data, serial = load_current(fs, oid)
        refs = get_refs(data)
        missing = []  # contains 3-tuples of oid, klass-metadata, reason
        for ref, klass in refs:
            if klass is None:
                klass = '<unknown>'
            if ref not in fs._index:
                missing.append((ref, klass, "missing"))
            if ref in noload:
                missing.append((ref, klass, "failed to load"))
            if ref in undone:
                missing.append((ref, klass, "object creation was undone"))
        if missing:
            report(oid, data, serial, missing)