Esempio n. 1
0
    def testDeepCopyCanInvalidate(self):
        """
        Tests regression for invalidation problems related to missing
        readers and writers values in cloned objects (see
        http://mail.zope.org/pipermail/zodb-dev/2008-August/012054.html)
        """
        import ZODB.MappingStorage
        database = DB(ZODB.blob.BlobStorage(
            'blobs', ZODB.MappingStorage.MappingStorage()))
        connection = database.open()
        root = connection.root()
        transaction.begin()
        root['blob'] = Blob()
        transaction.commit()

        stream = BytesIO()
        p = Pickler(stream, _protocol)
        p.dump(root['blob'])
        u = Unpickler(stream)
        stream.seek(0)
        clone = u.load()
        clone._p_invalidate()

        # it should also be possible to open the cloned blob
        # (even though it won't contain the original data)
        clone.open().close()

        # tearDown
        database.close()
Esempio n. 2
0
def zodb_unpickle(data):
    """Unpickle an object stored using the format expected by ZODB."""
    f = BytesIO(data)
    u = Unpickler(f)
    u.persistent_load = persistent_load
    klass_info = u.load()
    if isinstance(klass_info, tuple):
        if isinstance(klass_info[0], type):
            # Unclear:  what is the second part of klass_info?
            klass, xxx = klass_info
            assert not xxx
        else:
            if isinstance(klass_info[0], tuple):
                modname, klassname = klass_info[0]
            else:
                modname, klassname = klass_info
            if modname == "__main__":
                ns = globals()
            else:
                mod = import_helper(modname)
                ns = mod.__dict__
            try:
                klass = ns[klassname]
            except KeyError:
                print("can't find %s in %r" % (klassname, ns), file=sys.stderr)
        inst = klass()
    else:
        raise ValueError("expected class info: %s" % repr(klass_info))
    state = u.load()
    inst.__setstate__(state)
    return inst
def cloneByPickle(obj, ignore_list=()):
    """Makes a copy of a ZODB object, loading ghosts as needed.

    Ignores specified objects along the way, replacing them with None
    in the copy.
    """
    ignore_dict = {}
    for o in ignore_list:
        ignore_dict[id(o)] = o

    def persistent_id(ob, ignore_dict=ignore_dict):
        if id(ob) in ignore_dict:
            return 'ignored'
        if getattr(ob, '_p_changed', 0) is None:
            ob._p_changed = 0
        return None

    def persistent_load(ref):
        assert ref == 'ignored'
        # Return a placeholder object that will be replaced by
        # removeNonVersionedData().
        placeholder = SimpleItem()
        placeholder.id = "ignored_subobject"
        return placeholder

    stream = BytesIO()
    p = Pickler(stream, 1)
    p.persistent_id = persistent_id
    p.dump(obj)
    stream.seek(0)
    u = Unpickler(stream)
    u.persistent_load = persistent_load
    return u.load()
Esempio n. 4
0
def state(self, oid, serial, prfactory, p=''):
    p = p or self.loadSerial(oid, serial)
    p = self._crs_untransform_record_data(p)
    file = BytesIO(p)
    unpickler = Unpickler(file)
    unpickler.find_global = find_global
    unpickler.persistent_load = prfactory.persistent_load
    unpickler.load() # skip the class tuple
    return unpickler.load()
    def _get_unpickler(self, pickle):
        file = BytesIO(pickle)
        unpickler = Unpickler(file)
        unpickler.persistent_load = self._persistent_load
        factory = self._factory
        conn = self._conn

        def find_global(modulename, name):
            return factory(conn, modulename, name)

        unpickler.find_global = find_global

        return unpickler
Esempio n. 6
0
def get_pickle_metadata(data):
    # Returns a 2-tuple of strings.

    # ZODB's data records contain two pickles.  The first is the class
    # of the object, the second is the object.  We're only trying to
    # pick apart the first here, to extract the module and class names.
    if data[0] in (
            0x80,  # Py3k indexes bytes -> int
            b'\x80'  # Python2 indexes bytes -> bytes
    ):  # protocol marker, protocol > 1
        data = data[2:]
    if data.startswith(b'(c'):  # pickle MARK GLOBAL opcode sequence
        global_prefix = 2
    elif data.startswith(b'c'):  # pickle GLOBAL opcode
        global_prefix = 1
    else:
        global_prefix = 0

    if global_prefix:
        # Formats 1 and 2.
        # Don't actually unpickle a class, because it will attempt to
        # load the class.  Just break open the pickle and get the
        # module and class from it.  The module and class names are given by
        # newline-terminated strings following the GLOBAL opcode.
        modname, classname, rest = data.split(b'\n', 2)
        modname = modname[global_prefix:]  # strip GLOBAL opcode
        return modname.decode(), classname.decode()

    # Else there are a bunch of other possible formats.
    f = BytesIO(data)
    u = Unpickler(f)
    try:
        class_info = u.load()
    except Exception as err:
        return '', ''
    if isinstance(class_info, tuple):
        if isinstance(class_info[0], tuple):
            # Formats 3 and 4.
            modname, classname = class_info[0]
        else:
            # Formats 5 and 6 (probably) end up here.
            modname, classname = class_info
    else:
        # This isn't a known format.
        modname = repr(class_info)
        classname = ''
    return modname, classname
Esempio n. 7
0
def reorderPickle(jar, p):
    try:
        from ZODB._compat import Unpickler, Pickler
    except ImportError:  # BBB: ZODB 3.10
        from ZODB.ExportImport import Unpickler, Pickler
    from ZODB.ExportImport import Ghost, persistent_id

    oids = {}
    storage = jar._storage
    new_oid = storage.new_oid
    store = storage.store

    def persistent_load(ooid,
                        Ghost=Ghost,
                        oids=oids,
                        wrote_oid=oids.has_key,
                        new_oid=storage.new_oid):

        "Remap a persistent id to an existing ID and create a ghost for it."

        if type(ooid) is TupleType: ooid, klass = ooid
        else: klass = None

        try:
            Ghost = Ghost()
            Ghost.oid = ooid
        except TypeError:
            Ghost = Ghost(ooid)
        return Ghost

    # Reorder pickle by doing I/O
    pfile = StringIO(p)
    unpickler = Unpickler(pfile)
    unpickler.persistent_load = persistent_load

    newp = StringIO()
    pickler = OrderedPickler(newp, 1)
    pickler.persistent_id = persistent_id

    classdef = unpickler.load()
    obj = unpickler.load()
    pickler.dump(classdef)
    pickler.dump(obj)
    p = newp.getvalue()
    return obj, p
Esempio n. 8
0
 def load(class_, fname):
     with open(fname, 'rb') as f:
         unpickler = Unpickler(f)
         pos = unpickler.load()
         if not isinstance(pos, INT_TYPES):
             # NB: this might contain OIDs that got unpickled
             # into Unicode strings on Python 3; hope the caller
             # will pipe the result to fsIndex().update() to normalize
             # the keys
             return pos                  # Old format
         index = class_()
         data = index._data
         while 1:
             v = unpickler.load()
             if not v:
                 break
             k, v = v
             data[ensure_bytes(k)] = fsBucket().fromString(ensure_bytes(v))
         return dict(pos=pos, index=index)
Esempio n. 9
0
def is_blob_record(record):
    """Check whether a database record is a blob record.

    This is primarily intended to be used when copying data from one
    storage to another.

    """
    if record and (b'ZODB.blob' in record):
        unpickler = Unpickler(BytesIO(record))
        unpickler.find_global = find_global_Blob

        try:
            return unpickler.load() is Blob
        except (MemoryError, KeyboardInterrupt, SystemExit):
            raise
        except Exception:
            pass

    return False
def referencesf(p, oids=None):
    """Return a list of object ids found in a pickle

    A list may be passed in, in which case, information is
    appended to it.

    Only ordinary internal references are included.
    Weak and multi-database references are not included.
    """

    refs = []
    u = Unpickler(BytesIO(p))
    u.persistent_load = refs.append
    u.noload()
    u.noload()

    # Now we have a list of referencs.  Need to convert to list of
    # oids:

    if oids is None:
        oids = []

    for reference in refs:
        if isinstance(reference, tuple):
            oid = reference[0]
        elif isinstance(reference, (bytes, str)):
            oid = reference
        else:
            assert isinstance(reference, list)
            continue

        if not isinstance(oid, bytes):
            assert isinstance(oid, str)
            # this happens on Python 3 when all bytes in the oid are < 0x80
            oid = oid.encode('ascii')

        oids.append(oid)

    return oids
    def TODO_checkNewPicklesAreSafe(self):
        s = MappingStorage()
        db = ZODB.DB(s)
        r = db.open().root()
        r[1] = 1
        r[2] = 2
        r[3] = r
        transaction.commit()
        # MappingStorage stores serialno + pickle in its _index.
        root_pickle = s._index['\000' * 8][8:]

        # XXX not BytesIO really?
        f = cStringIO.StringIO(root_pickle)
        u = Unpickler(f)
        klass_info = u.load()
        klass = find_global(*klass_info[0])
        inst = klass.__new__(klass)
        state = u.load()
        inst.__setstate__(state)

        self.assertTrue(hasattr(inst, '_container'))
        self.assertTrue(not hasattr(inst, 'data'))
def get_refs(a_pickle):
    """Return oid and class information for references in a pickle

    The result of a list of oid and class information tuples.
    If the reference doesn't contain class information, then the
    klass information is None.
    """

    refs = []
    u = Unpickler(BytesIO(a_pickle))
    u.persistent_load = refs.append
    u.noload()
    u.noload()

    # Now we have a list of references.  Need to convert to list of
    # oids and class info:

    result = []

    for reference in refs:
        if isinstance(reference, tuple):
            oid, klass = reference
        elif isinstance(reference, (bytes, str)):
            data, klass = reference, None
        else:
            assert isinstance(reference, list)
            continue

        if not isinstance(oid, bytes):
            assert isinstance(oid, str)
            # this happens on Python 3 when all bytes in the oid are < 0x80
            oid = oid.encode('ascii')

        result.append((oid, klass))

    return result
Esempio n. 13
0
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
                         committedData=b''):
    # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
    try:
        prfactory = PersistentReferenceFactory()
        newpickle = self._crs_untransform_record_data(newpickle)
        file = BytesIO(newpickle)
        unpickler = Unpickler(file)
        unpickler.find_global = find_global
        unpickler.persistent_load = prfactory.persistent_load
        meta = unpickler.load()
        if isinstance(meta, tuple):
            klass = meta[0]
            newargs = meta[1] or ()
            if isinstance(klass, tuple):
                klass = find_global(*klass)
        else:
            klass = meta
            newargs = ()

        if klass in _unresolvable:
            raise ConflictError

        inst = klass.__new__(klass, *newargs)

        try:
            resolve = inst._p_resolveConflict
        except AttributeError:
            _unresolvable[klass] = 1
            raise ConflictError


        oldData = self.loadSerial(oid, oldSerial)
        if not committedData:
            committedData  = self.loadSerial(oid, committedSerial)

        if newpickle == oldData:
            # old -> new diff is empty, so merge is trivial
            return committedData
        if committedData == oldData:
            # old -> committed diff is empty, so merge is trivial
            return newpickle

        newstate = unpickler.load()
        old       = state(self, oid, oldSerial, prfactory, oldData)
        committed = state(self, oid, committedSerial, prfactory, committedData)

        resolved = resolve(old, committed, newstate)

        file = BytesIO()
        pickler = Pickler(file, _protocol)
        if sys.version_info[0] < 3:
            pickler.inst_persistent_id = persistent_id
        else:
            pickler.persistent_id = persistent_id
        pickler.dump(meta)
        pickler.dump(resolved)
        return self._crs_transform_record_data(file.getvalue())
    except (ConflictError, BadClassName):
        pass
    except:
        # If anything else went wrong, catch it here and avoid passing an
        # arbitrary exception back to the client.  The error here will mask
        # the original ConflictError.  A client can recover from a
        # ConflictError, but not necessarily from other errors.  But log
        # the error so that any problems can be fixed.
        logger.error("Unexpected error", exc_info=True)

    raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
                        data=newpickle)
Esempio n. 14
0
def unpickle_state(data):
    unpickler = Unpickler(StringIO(data))
    unpickler.persistent_load = PersistentReferenceFactory().persistent_load
    unpickler.load()  # skip the class tuple
    return unpickler.load()
Esempio n. 15
0
 def loads(str, persfunc=self._cache.get):
     fp = BytesIO(str)
     u = Unpickler(fp)
     u.persistent_load = persfunc
     return u.load()
Esempio n. 16
0
    def _importDuringCommit(self, transaction, f, return_oid_list):
        """Import data during two-phase commit.

        Invoked by the transaction manager mid commit.
        Appends one item, the OID of the first object created,
        to return_oid_list.
        """
        oids = {}

        # IMPORTANT: This code should be consistent with the code in
        # serialize.py. It is currently out of date and doesn't handle
        # weak references.

        def persistent_load(ooid):
            """Remap a persistent id to a new ID and create a ghost for it."""

            klass = None
            if isinstance(ooid, tuple):
                ooid, klass = ooid

            if not isinstance(ooid, bytes):
                assert isinstance(ooid, str)
                # this happens on Python 3 when all bytes in the oid are < 0x80
                ooid = ooid.encode('ascii')

            if ooid in oids:
                oid = oids[ooid]
            else:
                if klass is None:
                    oid = self._storage.new_oid()
                else:
                    oid = self._storage.new_oid(), klass
                oids[ooid] = oid

            return Ghost(oid)

        while 1:
            header = f.read(16)
            if header == export_end_marker:
                break
            if len(header) != 16:
                raise ExportError("Truncated export file")

            # Extract header information
            ooid = header[:8]
            length = u64(header[8:16])
            data = f.read(length)

            if len(data) != length:
                raise ExportError("Truncated export file")

            if oids:
                oid = oids[ooid]
                if isinstance(oid, tuple):
                    oid = oid[0]
            else:
                oids[ooid] = oid = self._storage.new_oid()
                return_oid_list.append(oid)

            # Blob support
            blob_begin = f.read(len(blob_begin_marker))
            if blob_begin == blob_begin_marker:
                # Copy the blob data to a temporary file
                # and remember the name
                blob_len = u64(f.read(8))
                blob_filename = mktemp()
                blob_file = open(blob_filename, "wb")
                cp(f, blob_file, blob_len)
                blob_file.close()
            else:
                f.seek(-len(blob_begin_marker), 1)
                blob_filename = None

            pfile = BytesIO(data)
            unpickler = Unpickler(pfile)
            unpickler.persistent_load = persistent_load

            newp = BytesIO()
            pickler = PersistentPickler(persistent_id, newp, _protocol)

            pickler.dump(unpickler.load())
            pickler.dump(unpickler.load())
            data = newp.getvalue()

            if blob_filename is not None:
                self._storage.storeBlob(oid, None, data, blob_filename, '',
                                        transaction)
            else:
                self._storage.store(oid, None, data, '', transaction)
Esempio n. 17
0
def FakeUnpickler(f):
    unpickler = Unpickler(f)
    unpickler.find_global = fake_find_class
    return unpickler