Beispiel #1
0
def zodb_unpickle(data):
    """Unpickle an object stored using the format expected by ZODB."""
    f = BytesIO(data)
    u = Unpickler(f)
    u.persistent_load = persistent_load
    klass_info = u.load()
    if isinstance(klass_info, tuple):
        if isinstance(klass_info[0], type):
            # Unclear:  what is the second part of klass_info?
            klass, xxx = klass_info
            assert not xxx
        else:
            if isinstance(klass_info[0], tuple):
                modname, klassname = klass_info[0]
            else:
                modname, klassname = klass_info
            if modname == "__main__":
                ns = globals()
            else:
                mod = import_helper(modname)
                ns = mod.__dict__
            try:
                klass = ns[klassname]
            except KeyError:
                print("can't find %s in %r" % (klassname, ns), file=sys.stderr)
        inst = klass()
    else:
        raise ValueError("expected class info: %s" % repr(klass_info))
    state = u.load()
    inst.__setstate__(state)
    return inst
def cloneByPickle(obj, ignore_list=()):
    """Makes a copy of a ZODB object, loading ghosts as needed.

    Ignores specified objects along the way, replacing them with None
    in the copy.
    """
    ignore_dict = {}
    for o in ignore_list:
        ignore_dict[id(o)] = o

    def persistent_id(ob, ignore_dict=ignore_dict):
        if id(ob) in ignore_dict:
            return 'ignored'
        if getattr(ob, '_p_changed', 0) is None:
            ob._p_changed = 0
        return None

    def persistent_load(ref):
        assert ref == 'ignored'
        # Return a placeholder object that will be replaced by
        # removeNonVersionedData().
        placeholder = SimpleItem()
        placeholder.id = "ignored_subobject"
        return placeholder

    stream = BytesIO()
    p = Pickler(stream, 1)
    p.persistent_id = persistent_id
    p.dump(obj)
    stream.seek(0)
    u = Unpickler(stream)
    u.persistent_load = persistent_load
    return u.load()
def cloneByPickle(obj, ignore_list=()):
    """Makes a copy of a ZODB object, loading ghosts as needed.

    Ignores specified objects along the way, replacing them with None
    in the copy.
    """
    ignore_dict = {}
    for o in ignore_list:
        ignore_dict[id(o)] = o

    def persistent_id(ob, ignore_dict=ignore_dict):
        if id(ob) in ignore_dict:
            return 'ignored'
        if getattr(ob, '_p_changed', 0) is None:
            ob._p_changed = 0
        return None

    def persistent_load(ref):
        assert ref == 'ignored'
        # Return a placeholder object that will be replaced by
        # removeNonVersionedData().
        placeholder = SimpleItem()
        placeholder.id = "ignored_subobject"
        return placeholder

    stream = BytesIO()
    p = Pickler(stream, 1)
    p.persistent_id = persistent_id
    p.dump(obj)
    stream.seek(0)
    u = Unpickler(stream)
    u.persistent_load = persistent_load
    return u.load()
Beispiel #4
0
def state(self, oid, serial, prfactory, p=''):
    p = p or self.loadSerial(oid, serial)
    p = self._crs_untransform_record_data(p)
    file = BytesIO(p)
    unpickler = Unpickler(file)
    unpickler.find_global = find_global
    unpickler.persistent_load = prfactory.persistent_load
    unpickler.load() # skip the class tuple
    return unpickler.load()
Beispiel #5
0
    def _get_unpickler(self, pickle):
        file = BytesIO(pickle)
        unpickler = Unpickler(file)
        unpickler.persistent_load = self._persistent_load
        factory = self._factory
        conn = self._conn

        def find_global(modulename, name):
            return factory(conn, modulename, name)

        unpickler.find_global = find_global

        return unpickler
    def _get_unpickler(self, pickle):
        file = BytesIO(pickle)
        unpickler = Unpickler(file)
        unpickler.persistent_load = self._persistent_load
        factory = self._factory
        conn = self._conn

        def find_global(modulename, name):
            return factory(conn, modulename, name)

        unpickler.find_global = find_global

        return unpickler
Beispiel #7
0
def reorderPickle(jar, p):
    try:
        from ZODB._compat import Unpickler, Pickler
    except ImportError:  # BBB: ZODB 3.10
        from ZODB.ExportImport import Unpickler, Pickler
    from ZODB.ExportImport import Ghost, persistent_id

    oids = {}
    storage = jar._storage
    new_oid = storage.new_oid
    store = storage.store

    def persistent_load(ooid,
                        Ghost=Ghost,
                        oids=oids,
                        wrote_oid=oids.has_key,
                        new_oid=storage.new_oid):

        "Remap a persistent id to an existing ID and create a ghost for it."

        if type(ooid) is TupleType: ooid, klass = ooid
        else: klass = None

        try:
            Ghost = Ghost()
            Ghost.oid = ooid
        except TypeError:
            Ghost = Ghost(ooid)
        return Ghost

    # Reorder pickle by doing I/O
    pfile = StringIO(p)
    unpickler = Unpickler(pfile)
    unpickler.persistent_load = persistent_load

    newp = StringIO()
    pickler = OrderedPickler(newp, 1)
    pickler.persistent_id = persistent_id

    classdef = unpickler.load()
    obj = unpickler.load()
    pickler.dump(classdef)
    pickler.dump(obj)
    p = newp.getvalue()
    return obj, p
Beispiel #8
0
def reorderPickle(jar, p):
    try:
        from ZODB._compat import Unpickler, Pickler
    except ImportError: # BBB: ZODB 3.10
        from ZODB.ExportImport import Unpickler, Pickler
    from ZODB.ExportImport import Ghost, persistent_id

    oids = {}
    storage = jar._storage
    new_oid = storage.new_oid
    store = storage.store

    def persistent_load(ooid,
                        Ghost=Ghost,
                        oids=oids, wrote_oid=oids.has_key,
                        new_oid=storage.new_oid):

        "Remap a persistent id to an existing ID and create a ghost for it."

        if type(ooid) is TupleType: ooid, klass = ooid
        else: klass=None

        try:
          Ghost=Ghost()
          Ghost.oid=ooid
        except TypeError:
          Ghost=Ghost(ooid)
        return Ghost

    # Reorder pickle by doing I/O
    pfile = StringIO(p)
    unpickler=Unpickler(pfile)
    unpickler.persistent_load=persistent_load

    newp=StringIO()
    pickler=OrderedPickler(newp,1)
    pickler.persistent_id=persistent_id

    classdef = unpickler.load()
    obj = unpickler.load()
    pickler.dump(classdef)
    pickler.dump(obj)
    p=newp.getvalue()
    return obj, p
Beispiel #9
0
def referencesf(p, oids=None):
    """Return a list of object ids found in a pickle

    A list may be passed in, in which case, information is
    appended to it.

    Only ordinary internal references are included.
    Weak and multi-database references are not included.
    """

    refs = []
    u = Unpickler(BytesIO(p))
    u.persistent_load = refs.append
    u.noload()
    u.noload()

    # Now we have a list of referencs.  Need to convert to list of
    # oids:

    if oids is None:
        oids = []

    for reference in refs:
        if isinstance(reference, tuple):
            oid = reference[0]
        elif isinstance(reference, (bytes, str)):
            oid = reference
        else:
            assert isinstance(reference, list)
            continue

        if not isinstance(oid, bytes):
            assert isinstance(oid, str)
            # this happens on Python 3 when all bytes in the oid are < 0x80
            oid = oid.encode("ascii")

        oids.append(oid)

    return oids
def referencesf(p, oids=None):
    """Return a list of object ids found in a pickle

    A list may be passed in, in which case, information is
    appended to it.

    Only ordinary internal references are included.
    Weak and multi-database references are not included.
    """

    refs = []
    u = Unpickler(BytesIO(p))
    u.persistent_load = refs.append
    u.noload()
    u.noload()

    # Now we have a list of referencs.  Need to convert to list of
    # oids:

    if oids is None:
        oids = []

    for reference in refs:
        if isinstance(reference, tuple):
            oid = reference[0]
        elif isinstance(reference, (bytes, str)):
            oid = reference
        else:
            assert isinstance(reference, list)
            continue

        if not isinstance(oid, bytes):
            assert isinstance(oid, str)
            # this happens on Python 3 when all bytes in the oid are < 0x80
            oid = oid.encode('ascii')

        oids.append(oid)

    return oids
Beispiel #11
0
def get_refs(a_pickle):
    """Return oid and class information for references in a pickle

    The result of a list of oid and class information tuples.
    If the reference doesn't contain class information, then the
    klass information is None.
    """

    refs = []
    u = Unpickler(BytesIO(a_pickle))
    u.persistent_load = refs.append
    u.noload()
    u.noload()

    # Now we have a list of references.  Need to convert to list of
    # oids and class info:

    result = []

    for reference in refs:
        if isinstance(reference, tuple):
            oid, klass = reference
        elif isinstance(reference, (bytes, str)):
            data, klass = reference, None
        else:
            assert isinstance(reference, list)
            continue

        if not isinstance(oid, bytes):
            assert isinstance(oid, str)
            # this happens on Python 3 when all bytes in the oid are < 0x80
            oid = oid.encode("ascii")

        result.append((oid, klass))

    return result
def get_refs(a_pickle):
    """Return oid and class information for references in a pickle

    The result of a list of oid and class information tuples.
    If the reference doesn't contain class information, then the
    klass information is None.
    """

    refs = []
    u = Unpickler(BytesIO(a_pickle))
    u.persistent_load = refs.append
    u.noload()
    u.noload()

    # Now we have a list of references.  Need to convert to list of
    # oids and class info:

    result = []

    for reference in refs:
        if isinstance(reference, tuple):
            oid, klass = reference
        elif isinstance(reference, (bytes, str)):
            data, klass = reference, None
        else:
            assert isinstance(reference, list)
            continue

        if not isinstance(oid, bytes):
            assert isinstance(oid, str)
            # this happens on Python 3 when all bytes in the oid are < 0x80
            oid = oid.encode('ascii')

        result.append((oid, klass))

    return result
Beispiel #13
0
 def loads(str, persfunc=self._cache.get):
     fp = BytesIO(str)
     u = Unpickler(fp)
     u.persistent_load = persfunc
     return u.load()
Beispiel #14
0
def unpickle_state(data):
    unpickler = Unpickler(StringIO(data))
    unpickler.persistent_load = PersistentReferenceFactory().persistent_load
    unpickler.load()  # skip the class tuple
    return unpickler.load()
Beispiel #15
0
    def _importDuringCommit(self, transaction, f, return_oid_list):
        """Import data during two-phase commit.

        Invoked by the transaction manager mid commit.
        Appends one item, the OID of the first object created,
        to return_oid_list.
        """
        oids = {}

        # IMPORTANT: This code should be consistent with the code in
        # serialize.py. It is currently out of date and doesn't handle
        # weak references.

        def persistent_load(ooid):
            """Remap a persistent id to a new ID and create a ghost for it."""

            klass = None
            if isinstance(ooid, tuple):
                ooid, klass = ooid

            if not isinstance(ooid, bytes):
                assert isinstance(ooid, str)
                # this happens on Python 3 when all bytes in the oid are < 0x80
                ooid = ooid.encode('ascii')

            if ooid in oids:
                oid = oids[ooid]
            else:
                if klass is None:
                    oid = self._storage.new_oid()
                else:
                    oid = self._storage.new_oid(), klass
                oids[ooid] = oid

            return Ghost(oid)

        while 1:
            header = f.read(16)
            if header == export_end_marker:
                break
            if len(header) != 16:
                raise ExportError("Truncated export file")

            # Extract header information
            ooid = header[:8]
            length = u64(header[8:16])
            data = f.read(length)

            if len(data) != length:
                raise ExportError("Truncated export file")

            if oids:
                oid = oids[ooid]
                if isinstance(oid, tuple):
                    oid = oid[0]
            else:
                oids[ooid] = oid = self._storage.new_oid()
                return_oid_list.append(oid)

            # Blob support
            blob_begin = f.read(len(blob_begin_marker))
            if blob_begin == blob_begin_marker:
                # Copy the blob data to a temporary file
                # and remember the name
                blob_len = u64(f.read(8))
                blob_filename = mktemp()
                blob_file = open(blob_filename, "wb")
                cp(f, blob_file, blob_len)
                blob_file.close()
            else:
                f.seek(-len(blob_begin_marker),1)
                blob_filename = None

            pfile = BytesIO(data)
            unpickler = Unpickler(pfile)
            unpickler.persistent_load = persistent_load

            newp = BytesIO()
            pickler = PersistentPickler(persistent_id, newp, _protocol)

            pickler.dump(unpickler.load())
            pickler.dump(unpickler.load())
            data = newp.getvalue()

            if blob_filename is not None:
                self._storage.storeBlob(oid, None, data, blob_filename,
                                        '', transaction)
            else:
                self._storage.store(oid, None, data, '', transaction)
Beispiel #16
0
 def loads(str, persfunc=self._cache.get):
     fp = BytesIO(str)
     u = Unpickler(fp)
     u.persistent_load = persfunc
     return u.load()
Beispiel #17
0
    def _importDuringCommit(self, transaction, f, return_oid_list):
        """Import data during two-phase commit.

        Invoked by the transaction manager mid commit.
        Appends one item, the OID of the first object created,
        to return_oid_list.
        """
        oids = {}

        # IMPORTANT: This code should be consistent with the code in
        # serialize.py. It is currently out of date and doesn't handle
        # weak references.

        def persistent_load(ooid):
            """Remap a persistent id to a new ID and create a ghost for it."""

            klass = None
            if isinstance(ooid, tuple):
                ooid, klass = ooid

            if not isinstance(ooid, bytes):
                assert isinstance(ooid, str)
                # this happens on Python 3 when all bytes in the oid are < 0x80
                ooid = ooid.encode('ascii')

            if ooid in oids:
                oid = oids[ooid]
            else:
                if klass is None:
                    oid = self._storage.new_oid()
                else:
                    oid = self._storage.new_oid(), klass
                oids[ooid] = oid

            return Ghost(oid)

        while 1:
            header = f.read(16)
            if header == export_end_marker:
                break
            if len(header) != 16:
                raise ExportError("Truncated export file")

            # Extract header information
            ooid = header[:8]
            length = u64(header[8:16])
            data = f.read(length)

            if len(data) != length:
                raise ExportError("Truncated export file")

            if oids:
                oid = oids[ooid]
                if isinstance(oid, tuple):
                    oid = oid[0]
            else:
                oids[ooid] = oid = self._storage.new_oid()
                return_oid_list.append(oid)

            # Blob support
            blob_begin = f.read(len(blob_begin_marker))
            if blob_begin == blob_begin_marker:
                # Copy the blob data to a temporary file
                # and remember the name
                blob_len = u64(f.read(8))
                blob_filename = mktemp()
                blob_file = open(blob_filename, "wb")
                cp(f, blob_file, blob_len)
                blob_file.close()
            else:
                f.seek(-len(blob_begin_marker), 1)
                blob_filename = None

            pfile = BytesIO(data)
            unpickler = Unpickler(pfile)
            unpickler.persistent_load = persistent_load

            newp = BytesIO()
            pickler = PersistentPickler(persistent_id, newp, _protocol)

            pickler.dump(unpickler.load())
            pickler.dump(unpickler.load())
            data = newp.getvalue()

            if blob_filename is not None:
                self._storage.storeBlob(oid, None, data, blob_filename, '',
                                        transaction)
            else:
                self._storage.store(oid, None, data, '', transaction)
Beispiel #18
0
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
                         committedData=b''):
    # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
    try:
        prfactory = PersistentReferenceFactory()
        newpickle = self._crs_untransform_record_data(newpickle)
        file = BytesIO(newpickle)
        unpickler = Unpickler(file)
        unpickler.find_global = find_global
        unpickler.persistent_load = prfactory.persistent_load
        meta = unpickler.load()
        if isinstance(meta, tuple):
            klass = meta[0]
            newargs = meta[1] or ()
            if isinstance(klass, tuple):
                klass = find_global(*klass)
        else:
            klass = meta
            newargs = ()

        if klass in _unresolvable:
            raise ConflictError

        inst = klass.__new__(klass, *newargs)

        try:
            resolve = inst._p_resolveConflict
        except AttributeError:
            _unresolvable[klass] = 1
            raise ConflictError


        oldData = self.loadSerial(oid, oldSerial)
        if not committedData:
            committedData  = self.loadSerial(oid, committedSerial)

        if newpickle == oldData:
            # old -> new diff is empty, so merge is trivial
            return committedData
        if committedData == oldData:
            # old -> committed diff is empty, so merge is trivial
            return newpickle

        newstate = unpickler.load()
        old       = state(self, oid, oldSerial, prfactory, oldData)
        committed = state(self, oid, committedSerial, prfactory, committedData)

        resolved = resolve(old, committed, newstate)

        file = BytesIO()
        pickler = Pickler(file, _protocol)
        if sys.version_info[0] < 3:
            pickler.inst_persistent_id = persistent_id
        else:
            pickler.persistent_id = persistent_id
        pickler.dump(meta)
        pickler.dump(resolved)
        return self._crs_transform_record_data(file.getvalue())
    except (ConflictError, BadClassName):
        pass
    except:
        # If anything else went wrong, catch it here and avoid passing an
        # arbitrary exception back to the client.  The error here will mask
        # the original ConflictError.  A client can recover from a
        # ConflictError, but not necessarily from other errors.  But log
        # the error so that any problems can be fixed.
        logger.error("Unexpected error", exc_info=True)

    raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
                        data=newpickle)