def __init__(self, converter_caps=None, pre_converter=None, registry=None, externalizer=None, source_ver=None, target_ver=None): global _global_registry assert ((source_ver is None) and (target_ver is None)) \ or ((source_ver is not None) and (target_ver is not None)) self.converter_capabilities = converter_caps or DEFAULT_CONVERTER_CAPS self._pre_converter = pre_converter and IConverter(pre_converter) self._registry = IRegistry(registry) if registry else _global_registry self._externalizer = externalizer and IExternalizer(externalizer) self._source_ver = source_ver self._target_ver = target_ver self.reset()
class Unserializer(object): """Base class for unserializers. It handle delayed unpacking and instance restoration to resolve circular references. If no registry instance is specified when created, the default global registry will be used. A pre-converter can be specified at creation time, if so the data will be first converted by the given converter and then the result will be unserialized. Used to parse data before unserializing. """ pass_through_types = () def __init__(self, converter_caps=None, pre_converter=None, registry=None, externalizer=None, source_ver=None, target_ver=None): global _global_registry assert ((source_ver is None) and (target_ver is None)) \ or ((source_ver is not None) and (target_ver is not None)) self.converter_capabilities = converter_caps or DEFAULT_CONVERTER_CAPS self._pre_converter = pre_converter and IConverter(pre_converter) self._registry = IRegistry(registry) if registry else _global_registry self._externalizer = externalizer and IExternalizer(externalizer) self._source_ver = source_ver self._target_ver = target_ver self.reset() ### IConverter ### def convert(self, data): try: # Pre-convert the data if a convertor was specified converted = self.pre_convertion(data) # Unpack the first level of values unpacked = self.unpack_data(converted) # Continue unpacking level by level self.finish_unpacking() # Inform object that it has migrated if this is a case if (IVersionAdapter.providedBy(unpacked) and self._migrated): unpacked.set_migrated() # Should be finished by now return unpacked finally: # Reset the state to cleanup all references self.reset() ### protected ### def pre_convertion(self, data): if self._pre_converter is not None: return self._pre_converter.convert(data) return data def reset(self): self._references = {} # {REFERENCE_ID: (DATA_ID, OBJECT)} self._pending = [] # Pendings unpacking self._instances = [] # [(RESTORATOR, INSTANCE, SNAPSHOT, REFID)] self._delayed = 0 # If we are in a delayable unpacking # If some snapshot has been migrated between versions self._migrated = False def unpack_data(self, data): return self._unpack_data(data, None, None) def delayed_unpacking(self, container, fun, *args, **kwargs): """Should be used when unpacking mutable values. This allows circular references resolution by pausing serialization.""" try: self._delayed += 1 blob = self._begin() try: fun(*args, **kwargs) self._commit(blob) return container except DelayPacking: self._rollback(blob) continuation = (fun, args, kwargs) self._pending.append(continuation) return container finally: self._delayed -= 1 def finish_unpacking(self): while self._pending: fun, args, kwargs = self._pending.pop(0) fun(*args, **kwargs) # Initialize delayed mutable instances in creation order for restorator, instance, snapshot, _refid in self._instances: if restorator is not None: # delayed mutable instances snapshot = self._adapt_snapshot(restorator, snapshot) instance.recover(snapshot) # Calls the instances post restoration callback in reversed order # in an intent to reduce the possibilities of instances relying # on there references being fully restored when called. # This should not be relied on anyway. for _, instance, _, _ in reversed(self._instances): restored_fun = getattr(instance, "restored", None) if restored_fun is not None: restored_fun() def restore_type(self, type_name): value = reflect.named_object(type_name) if issubclass(value, type): raise ValueError("type %r unserialized to something that " "isn't a type: %r" % (type_name, value)) return value def restore_external(self, data): if self._externalizer is None: raise ValueError("Got external reference %r but unserializer " "do not have any IExternalizer") identifier = self.unpack_data(data) instance = self._externalizer.lookup(identifier) if instance is None: raise ValueError("No external reference found with identifier %r" % (identifier, )) return instance def prepare_instance(self, type_name): restorator = self._lookup_restorator(type_name) # Prepare the instance for recovery instance = restorator.prepare() if instance is not None: return restorator, instance def restore_instance(self, type_name, data, refid=None, restorator=None, instance=None): if restorator is None: restorator = self._lookup_restorator(type_name) if instance is None: # Prepare the instance for recovery instance = restorator.prepare() if instance is None: # Immutable type, we can't delay restoration snapshot = self.unpack_data(data) snapshot = self._adapt_snapshot(restorator, snapshot) instance = restorator.restore(snapshot) self._instances.append((None, instance, None, refid)) return instance # Delay the instance restoration for later to handle circular refs return self.delayed_unpacking(instance, self._continue_restoring_instance, restorator, instance, data, refid) def restore_reference(self, refid, data): if refid in self._references: # This is because of DelayUnpacking exception, reference # can be registered multiple times data_id, value = self._references[refid] if data_id == id(data): return value raise ValueError("Multiple references found with " "the same identifier: %s" % refid) value = self._unpack_data(data, refid, data) if refid not in self._references: # If not yet referenced self._references[refid] = (id(data), value) return value def restore_dereference(self, refid): if refid not in self._references: # Dereference to an unknown reference if self._delayed > 0: # If we unpacking can be delayed because we are unpacking # a mutable object just delay the unpacking for later raise DelayPacking() raise ValueError("Dereferencing of yet unknown reference: %s" % refid) _data_id, value = self._references[refid] return value def unpack_unordered_values(self, values): """Unpack an unordered list of values taking DelayPacking exceptions into account to resolve circular references . Used to unpack set values when order is not guaranteed by the serializer. See unpack_unordered_pairs().""" values = list(values) # To support iterators result = [] # Try to unpack values more than one time to resolve cross references max_loop = 2 while values and max_loop: next_values = [] for value_data in values: blob = self._begin() try: # try unpacking the value value = self.unpack_data(value_data) self._commit(blob) except DelayPacking: self._rollback(blob) # If it is delayed keep it for later next_values.append(value_data) continue result.append(value) values = next_values max_loop -= 1 if values: # Not all items were resolved raise DelayPacking() return result def unpack_unordered_pairs(self, pairs): """Unpack an unordered list of value pairs taking DelayPacking exceptions into account to resolve circular references . Used to unpack dictionary items when the order is not guarennteed by the serializer. When item order change between packing and unpacking, references are not guaranteed to appear before dereferences anymore. So if unpacking an item fail because of unknown dereference, we must keep it aside, continue unpacking the other items and continue later.""" items = [(False, k, v) for k, v in pairs] result = [] # Try to unpack items more than one time to resolve cross references max_loop = 2 while items and max_loop: next_items = [] for key_unpacked, key_data, value_data in items: if key_unpacked: key = key_data else: blob = self._begin() try: # Try unpacking the key key = self.unpack_data(key_data) self._commit(blob) except DelayPacking: self._rollback(blob) # If it is delayed keep it for later next_items.append((False, key_data, value_data)) continue blob = self._begin() try: # try unpacking the value value = self.unpack_data(value_data) self._commit(blob) except DelayPacking: self._rollback(blob) # If it is delayed keep it for later next_items.append((True, key, value_data)) continue # Update the container with the unpacked value and key result.append((key, value)) items = next_items max_loop -= 1 if items: # Not all items were resolved raise DelayPacking() return result ### virtual ### def analyse_data(self, data): """Analyses the data provided and return a tuple containing the data type and a function to unpack it. The type can be None for immutable types, instances, reference and dereferences.""" ### private ### def _begin(self): # Start a DelayPacking protected section blob = self._instances self._instances = [] return blob def _rollback(self, blob): # We need to rollback after a DelayPacking has been raised # we only keep instances that has been referenced for instance in self._instances: refid = instance[3] if refid is not None: blob.append(instance) self._instances = blob def _commit(self, blob): # Commit after a DelayPacking protected section # Joining the instance lists blob.extend(self._instances) self._instances = blob def _lookup_restorator(self, type_name): # Lookup the registry for a IRestorator restorator = self._registry.lookup(type_name) if restorator is None: raise TypeError("Type %s not supported by unserializer %s" % (type_name, reflect.canonical_name(self))) return restorator def _unpack_data(self, data, refid, refdata): # Just return pass-through types, # support sub-classed base types and metaclasses if set(type(data).__mro__) & self.pass_through_types: return data analysis = self.analyse_data(data) if analysis is not None: constructor, unpacker = analysis if constructor is None: # Immutable types return unpacker(self, data) if callable(constructor): # Unpack the mutable containers that provides constructor container = constructor() if container is not None: if refid is not None: self._references[refid] = (id(refdata), container) return self.delayed_unpacking(container, unpacker, self, container, data) else: # Instance type name prepared = self.prepare_instance(constructor) if prepared is None: # Immutable instance return unpacker(self, data, None, None, None) restorator, instance = prepared if refid is not None: self._references[refid] = (id(refdata), instance) return self.delayed_unpacking(instance, unpacker, self, data, refid, restorator, instance) raise TypeError("Type %s not supported by unserializer %s" % (type(data).__name__, reflect.canonical_name(self))) def _continue_restoring_instance(self, restorator, instance, data, refid): snapshot = self.unpack_data(data) # Delay instance initialization to the end to be sure # all snapshots circular references have been resolved self._instances.append((restorator, instance, snapshot, refid)) return instance def _adapt_snapshot(self, restorator, snapshot): try: adapter = IVersionAdapter(restorator) except TypeError: pass else: target = self.get_target_ver(restorator, snapshot) source = self.get_source_ver(restorator, snapshot) if target is not None and target != source: snapshot = adapter.adapt_version(snapshot, source, target) return snapshot def get_target_ver(self, restorator, snapshot): return self._target_ver def get_source_ver(self, restorator, snapshot): return self._source_ver