def zodb_unpickle(data): """Unpickle an object stored using the format expected by ZODB.""" f = BytesIO(data) u = Unpickler(f) u.persistent_load = persistent_load klass_info = u.load() if isinstance(klass_info, tuple): if isinstance(klass_info[0], type): # Unclear: what is the second part of klass_info? klass, xxx = klass_info assert not xxx else: if isinstance(klass_info[0], tuple): modname, klassname = klass_info[0] else: modname, klassname = klass_info if modname == "__main__": ns = globals() else: mod = import_helper(modname) ns = mod.__dict__ try: klass = ns[klassname] except KeyError: print("can't find %s in %r" % (klassname, ns), file=sys.stderr) inst = klass() else: raise ValueError("expected class info: %s" % repr(klass_info)) state = u.load() inst.__setstate__(state) return inst
def cloneByPickle(obj, ignore_list=()): """Makes a copy of a ZODB object, loading ghosts as needed. Ignores specified objects along the way, replacing them with None in the copy. """ ignore_dict = {} for o in ignore_list: ignore_dict[id(o)] = o def persistent_id(ob, ignore_dict=ignore_dict): if id(ob) in ignore_dict: return 'ignored' if getattr(ob, '_p_changed', 0) is None: ob._p_changed = 0 return None def persistent_load(ref): assert ref == 'ignored' # Return a placeholder object that will be replaced by # removeNonVersionedData(). placeholder = SimpleItem() placeholder.id = "ignored_subobject" return placeholder stream = BytesIO() p = Pickler(stream, 1) p.persistent_id = persistent_id p.dump(obj) stream.seek(0) u = Unpickler(stream) u.persistent_load = persistent_load return u.load()
def state(self, oid, serial, prfactory, p=''): p = p or self.loadSerial(oid, serial) p = self._crs_untransform_record_data(p) file = BytesIO(p) unpickler = Unpickler(file) unpickler.find_global = find_global unpickler.persistent_load = prfactory.persistent_load unpickler.load() # skip the class tuple return unpickler.load()
def _get_unpickler(self, pickle): file = BytesIO(pickle) unpickler = Unpickler(file) unpickler.persistent_load = self._persistent_load factory = self._factory conn = self._conn def find_global(modulename, name): return factory(conn, modulename, name) unpickler.find_global = find_global return unpickler
def reorderPickle(jar, p): try: from ZODB._compat import Unpickler, Pickler except ImportError: # BBB: ZODB 3.10 from ZODB.ExportImport import Unpickler, Pickler from ZODB.ExportImport import Ghost, persistent_id oids = {} storage = jar._storage new_oid = storage.new_oid store = storage.store def persistent_load(ooid, Ghost=Ghost, oids=oids, wrote_oid=oids.has_key, new_oid=storage.new_oid): "Remap a persistent id to an existing ID and create a ghost for it." if type(ooid) is TupleType: ooid, klass = ooid else: klass = None try: Ghost = Ghost() Ghost.oid = ooid except TypeError: Ghost = Ghost(ooid) return Ghost # Reorder pickle by doing I/O pfile = StringIO(p) unpickler = Unpickler(pfile) unpickler.persistent_load = persistent_load newp = StringIO() pickler = OrderedPickler(newp, 1) pickler.persistent_id = persistent_id classdef = unpickler.load() obj = unpickler.load() pickler.dump(classdef) pickler.dump(obj) p = newp.getvalue() return obj, p
def reorderPickle(jar, p): try: from ZODB._compat import Unpickler, Pickler except ImportError: # BBB: ZODB 3.10 from ZODB.ExportImport import Unpickler, Pickler from ZODB.ExportImport import Ghost, persistent_id oids = {} storage = jar._storage new_oid = storage.new_oid store = storage.store def persistent_load(ooid, Ghost=Ghost, oids=oids, wrote_oid=oids.has_key, new_oid=storage.new_oid): "Remap a persistent id to an existing ID and create a ghost for it." if type(ooid) is TupleType: ooid, klass = ooid else: klass=None try: Ghost=Ghost() Ghost.oid=ooid except TypeError: Ghost=Ghost(ooid) return Ghost # Reorder pickle by doing I/O pfile = StringIO(p) unpickler=Unpickler(pfile) unpickler.persistent_load=persistent_load newp=StringIO() pickler=OrderedPickler(newp,1) pickler.persistent_id=persistent_id classdef = unpickler.load() obj = unpickler.load() pickler.dump(classdef) pickler.dump(obj) p=newp.getvalue() return obj, p
def referencesf(p, oids=None): """Return a list of object ids found in a pickle A list may be passed in, in which case, information is appended to it. Only ordinary internal references are included. Weak and multi-database references are not included. """ refs = [] u = Unpickler(BytesIO(p)) u.persistent_load = refs.append u.noload() u.noload() # Now we have a list of referencs. Need to convert to list of # oids: if oids is None: oids = [] for reference in refs: if isinstance(reference, tuple): oid = reference[0] elif isinstance(reference, (bytes, str)): oid = reference else: assert isinstance(reference, list) continue if not isinstance(oid, bytes): assert isinstance(oid, str) # this happens on Python 3 when all bytes in the oid are < 0x80 oid = oid.encode("ascii") oids.append(oid) return oids
def referencesf(p, oids=None): """Return a list of object ids found in a pickle A list may be passed in, in which case, information is appended to it. Only ordinary internal references are included. Weak and multi-database references are not included. """ refs = [] u = Unpickler(BytesIO(p)) u.persistent_load = refs.append u.noload() u.noload() # Now we have a list of referencs. Need to convert to list of # oids: if oids is None: oids = [] for reference in refs: if isinstance(reference, tuple): oid = reference[0] elif isinstance(reference, (bytes, str)): oid = reference else: assert isinstance(reference, list) continue if not isinstance(oid, bytes): assert isinstance(oid, str) # this happens on Python 3 when all bytes in the oid are < 0x80 oid = oid.encode('ascii') oids.append(oid) return oids
def get_refs(a_pickle): """Return oid and class information for references in a pickle The result of a list of oid and class information tuples. If the reference doesn't contain class information, then the klass information is None. """ refs = [] u = Unpickler(BytesIO(a_pickle)) u.persistent_load = refs.append u.noload() u.noload() # Now we have a list of references. Need to convert to list of # oids and class info: result = [] for reference in refs: if isinstance(reference, tuple): oid, klass = reference elif isinstance(reference, (bytes, str)): data, klass = reference, None else: assert isinstance(reference, list) continue if not isinstance(oid, bytes): assert isinstance(oid, str) # this happens on Python 3 when all bytes in the oid are < 0x80 oid = oid.encode("ascii") result.append((oid, klass)) return result
def get_refs(a_pickle): """Return oid and class information for references in a pickle The result of a list of oid and class information tuples. If the reference doesn't contain class information, then the klass information is None. """ refs = [] u = Unpickler(BytesIO(a_pickle)) u.persistent_load = refs.append u.noload() u.noload() # Now we have a list of references. Need to convert to list of # oids and class info: result = [] for reference in refs: if isinstance(reference, tuple): oid, klass = reference elif isinstance(reference, (bytes, str)): data, klass = reference, None else: assert isinstance(reference, list) continue if not isinstance(oid, bytes): assert isinstance(oid, str) # this happens on Python 3 when all bytes in the oid are < 0x80 oid = oid.encode('ascii') result.append((oid, klass)) return result
def loads(str, persfunc=self._cache.get): fp = BytesIO(str) u = Unpickler(fp) u.persistent_load = persfunc return u.load()
def unpickle_state(data): unpickler = Unpickler(StringIO(data)) unpickler.persistent_load = PersistentReferenceFactory().persistent_load unpickler.load() # skip the class tuple return unpickler.load()
def _importDuringCommit(self, transaction, f, return_oid_list): """Import data during two-phase commit. Invoked by the transaction manager mid commit. Appends one item, the OID of the first object created, to return_oid_list. """ oids = {} # IMPORTANT: This code should be consistent with the code in # serialize.py. It is currently out of date and doesn't handle # weak references. def persistent_load(ooid): """Remap a persistent id to a new ID and create a ghost for it.""" klass = None if isinstance(ooid, tuple): ooid, klass = ooid if not isinstance(ooid, bytes): assert isinstance(ooid, str) # this happens on Python 3 when all bytes in the oid are < 0x80 ooid = ooid.encode('ascii') if ooid in oids: oid = oids[ooid] else: if klass is None: oid = self._storage.new_oid() else: oid = self._storage.new_oid(), klass oids[ooid] = oid return Ghost(oid) while 1: header = f.read(16) if header == export_end_marker: break if len(header) != 16: raise ExportError("Truncated export file") # Extract header information ooid = header[:8] length = u64(header[8:16]) data = f.read(length) if len(data) != length: raise ExportError("Truncated export file") if oids: oid = oids[ooid] if isinstance(oid, tuple): oid = oid[0] else: oids[ooid] = oid = self._storage.new_oid() return_oid_list.append(oid) # Blob support blob_begin = f.read(len(blob_begin_marker)) if blob_begin == blob_begin_marker: # Copy the blob data to a temporary file # and remember the name blob_len = u64(f.read(8)) blob_filename = mktemp() blob_file = open(blob_filename, "wb") cp(f, blob_file, blob_len) blob_file.close() else: f.seek(-len(blob_begin_marker),1) blob_filename = None pfile = BytesIO(data) unpickler = Unpickler(pfile) unpickler.persistent_load = persistent_load newp = BytesIO() pickler = PersistentPickler(persistent_id, newp, _protocol) pickler.dump(unpickler.load()) pickler.dump(unpickler.load()) data = newp.getvalue() if blob_filename is not None: self._storage.storeBlob(oid, None, data, blob_filename, '', transaction) else: self._storage.store(oid, None, data, '', transaction)
def _importDuringCommit(self, transaction, f, return_oid_list): """Import data during two-phase commit. Invoked by the transaction manager mid commit. Appends one item, the OID of the first object created, to return_oid_list. """ oids = {} # IMPORTANT: This code should be consistent with the code in # serialize.py. It is currently out of date and doesn't handle # weak references. def persistent_load(ooid): """Remap a persistent id to a new ID and create a ghost for it.""" klass = None if isinstance(ooid, tuple): ooid, klass = ooid if not isinstance(ooid, bytes): assert isinstance(ooid, str) # this happens on Python 3 when all bytes in the oid are < 0x80 ooid = ooid.encode('ascii') if ooid in oids: oid = oids[ooid] else: if klass is None: oid = self._storage.new_oid() else: oid = self._storage.new_oid(), klass oids[ooid] = oid return Ghost(oid) while 1: header = f.read(16) if header == export_end_marker: break if len(header) != 16: raise ExportError("Truncated export file") # Extract header information ooid = header[:8] length = u64(header[8:16]) data = f.read(length) if len(data) != length: raise ExportError("Truncated export file") if oids: oid = oids[ooid] if isinstance(oid, tuple): oid = oid[0] else: oids[ooid] = oid = self._storage.new_oid() return_oid_list.append(oid) # Blob support blob_begin = f.read(len(blob_begin_marker)) if blob_begin == blob_begin_marker: # Copy the blob data to a temporary file # and remember the name blob_len = u64(f.read(8)) blob_filename = mktemp() blob_file = open(blob_filename, "wb") cp(f, blob_file, blob_len) blob_file.close() else: f.seek(-len(blob_begin_marker), 1) blob_filename = None pfile = BytesIO(data) unpickler = Unpickler(pfile) unpickler.persistent_load = persistent_load newp = BytesIO() pickler = PersistentPickler(persistent_id, newp, _protocol) pickler.dump(unpickler.load()) pickler.dump(unpickler.load()) data = newp.getvalue() if blob_filename is not None: self._storage.storeBlob(oid, None, data, blob_filename, '', transaction) else: self._storage.store(oid, None, data, '', transaction)
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle, committedData=b''): # class_tuple, old, committed, newstate = ('',''), 0, 0, 0 try: prfactory = PersistentReferenceFactory() newpickle = self._crs_untransform_record_data(newpickle) file = BytesIO(newpickle) unpickler = Unpickler(file) unpickler.find_global = find_global unpickler.persistent_load = prfactory.persistent_load meta = unpickler.load() if isinstance(meta, tuple): klass = meta[0] newargs = meta[1] or () if isinstance(klass, tuple): klass = find_global(*klass) else: klass = meta newargs = () if klass in _unresolvable: raise ConflictError inst = klass.__new__(klass, *newargs) try: resolve = inst._p_resolveConflict except AttributeError: _unresolvable[klass] = 1 raise ConflictError oldData = self.loadSerial(oid, oldSerial) if not committedData: committedData = self.loadSerial(oid, committedSerial) if newpickle == oldData: # old -> new diff is empty, so merge is trivial return committedData if committedData == oldData: # old -> committed diff is empty, so merge is trivial return newpickle newstate = unpickler.load() old = state(self, oid, oldSerial, prfactory, oldData) committed = state(self, oid, committedSerial, prfactory, committedData) resolved = resolve(old, committed, newstate) file = BytesIO() pickler = Pickler(file, _protocol) if sys.version_info[0] < 3: pickler.inst_persistent_id = persistent_id else: pickler.persistent_id = persistent_id pickler.dump(meta) pickler.dump(resolved) return self._crs_transform_record_data(file.getvalue()) except (ConflictError, BadClassName): pass except: # If anything else went wrong, catch it here and avoid passing an # arbitrary exception back to the client. The error here will mask # the original ConflictError. A client can recover from a # ConflictError, but not necessarily from other errors. But log # the error so that any problems can be fixed. logger.error("Unexpected error", exc_info=True) raise ConflictError(oid=oid, serials=(committedSerial, oldSerial), data=newpickle)