def __init__(self,
              converter_caps=None,
              pre_converter=None,
              registry=None,
              externalizer=None,
              source_ver=None,
              target_ver=None):
     global _global_registry
     assert ((source_ver is None) and (target_ver is None)) \
         or ((source_ver is not None) and (target_ver is not None))
     self.converter_capabilities = converter_caps or DEFAULT_CONVERTER_CAPS
     self._pre_converter = pre_converter and IConverter(pre_converter)
     self._registry = IRegistry(registry) if registry else _global_registry
     self._externalizer = externalizer and IExternalizer(externalizer)
     self._source_ver = source_ver
     self._target_ver = target_ver
     self.reset()
 def __init__(self, converter_caps=None, pre_converter=None,
              registry=None, externalizer=None,
              source_ver=None, target_ver=None):
     global _global_registry
     assert ((source_ver is None) and (target_ver is None)) \
         or ((source_ver is not None) and (target_ver is not None))
     self.converter_capabilities = converter_caps or DEFAULT_CONVERTER_CAPS
     self._pre_converter = pre_converter and IConverter(pre_converter)
     self._registry = IRegistry(registry) if registry else _global_registry
     self._externalizer = externalizer and IExternalizer(externalizer)
     self._source_ver = source_ver
     self._target_ver = target_ver
     self.reset()
class Unserializer(object):
    """Base class for unserializers. It handle delayed unpacking
    and instance restoration to resolve circular references.
    If no registry instance is specified when created, the default
    global registry will be used.

    A pre-converter can be specified at creation time, if so the
    data will be first converted by the given converter and then
    the result will be unserialized. Used to parse data before
    unserializing.
    """

    pass_through_types = ()

    def __init__(self, converter_caps=None, pre_converter=None,
                 registry=None, externalizer=None,
                 source_ver=None, target_ver=None):
        global _global_registry
        assert ((source_ver is None) and (target_ver is None)) \
            or ((source_ver is not None) and (target_ver is not None))
        self.converter_capabilities = converter_caps or DEFAULT_CONVERTER_CAPS
        self._pre_converter = pre_converter and IConverter(pre_converter)
        self._registry = IRegistry(registry) if registry else _global_registry
        self._externalizer = externalizer and IExternalizer(externalizer)
        self._source_ver = source_ver
        self._target_ver = target_ver
        self.reset()

    ### IConverter ###

    def convert(self, data):
        try:
            # Pre-convert the data if a convertor was specified
            converted = self.pre_convertion(data)
            # Unpack the first level of values
            unpacked = self.unpack_data(converted)
            # Continue unpacking level by level
            self.finish_unpacking()
            # Inform object that it has migrated if this is a case
            if (IVersionAdapter.providedBy(unpacked) and
                    self._migrated):
                unpacked.set_migrated()

            # Should be finished by now
            return unpacked
        finally:
            # Reset the state to cleanup all references
            self.reset()

    ### protected ###

    def pre_convertion(self, data):
        if self._pre_converter is not None:
            return self._pre_converter.convert(data)
        return data

    def reset(self):
        self._references = {}  # {REFERENCE_ID: (DATA_ID, OBJECT)}
        self._pending = []  # Pendings unpacking
        self._instances = []  # [(RESTORATOR, INSTANCE, SNAPSHOT, REFID)]
        self._delayed = 0  # If we are in a delayable unpacking
        # If some snapshot has been migrated between versions
        self._migrated = False

    def unpack_data(self, data):
        return self._unpack_data(data, None, None)

    def delayed_unpacking(self, container, fun, *args, **kwargs):
        """Should be used when unpacking mutable values.
        This allows circular references resolution by pausing serialization."""
        try:
            self._delayed += 1
            blob = self._begin()
            try:
                fun(*args, **kwargs)
                self._commit(blob)
                return container
            except DelayPacking:
                self._rollback(blob)
                continuation = (fun, args, kwargs)
                self._pending.append(continuation)
                return container
        finally:
            self._delayed -= 1

    def finish_unpacking(self):
        while self._pending:
            fun, args, kwargs = self._pending.pop(0)
            fun(*args, **kwargs)

        # Initialize delayed mutable instances in creation order
        for restorator, instance, snapshot, _refid in self._instances:
            if restorator is not None:
                # delayed mutable instances
                snapshot = self._adapt_snapshot(restorator, snapshot)
                instance.recover(snapshot)

        # Calls the instances post restoration callback in reversed order
        # in an intent to reduce the possibilities of instances relying
        # on there references being fully restored when called.
        # This should not be relied on anyway.
        for _, instance, _, _ in reversed(self._instances):
            restored_fun = getattr(instance, "restored", None)
            if restored_fun is not None:
                restored_fun()

    def restore_type(self, type_name):
        value = reflect.named_object(type_name)
        if issubclass(value, type):
            raise ValueError("type %r unserialized to something that "
                             "isn't a type: %r" % (type_name, value))
        return value

    def restore_external(self, data):
        if self._externalizer is None:
            raise ValueError("Got external reference %r but unserializer "
                             "do not have any IExternalizer")
        identifier = self.unpack_data(data)
        instance = self._externalizer.lookup(identifier)
        if instance is None:
            raise ValueError("No external reference found with identifier %r"
                             % (identifier, ))
        return instance

    def prepare_instance(self, type_name):
        restorator = self._lookup_restorator(type_name)
        # Prepare the instance for recovery
        instance = restorator.prepare()
        if instance is not None:
            return restorator, instance

    def restore_instance(self, type_name, data, refid=None,
                         restorator=None, instance=None):
        if restorator is None:
            restorator = self._lookup_restorator(type_name)

        if instance is None:
            # Prepare the instance for recovery
            instance = restorator.prepare()

        if instance is None:
            # Immutable type, we can't delay restoration
            snapshot = self.unpack_data(data)
            snapshot = self._adapt_snapshot(restorator, snapshot)
            instance = restorator.restore(snapshot)
            self._instances.append((None, instance, None, refid))

            return instance

        # Delay the instance restoration for later to handle circular refs
        return self.delayed_unpacking(instance,
                                      self._continue_restoring_instance,
                                      restorator, instance, data, refid)

    def restore_reference(self, refid, data):
        if refid in self._references:
            # This is because of DelayUnpacking exception, reference
            # can be registered multiple times
            data_id, value = self._references[refid]
            if data_id == id(data):
                return value
            raise ValueError("Multiple references found with "
                             "the same identifier: %s" % refid)
        value = self._unpack_data(data, refid, data)
        if refid not in self._references:
            # If not yet referenced
            self._references[refid] = (id(data), value)
        return value

    def restore_dereference(self, refid):
        if refid not in self._references:
            # Dereference to an unknown reference
            if self._delayed > 0:
                # If we unpacking can be delayed because we are unpacking
                # a mutable object just delay the unpacking for later
                raise DelayPacking()
            raise ValueError("Dereferencing of yet unknown reference: %s"
                             % refid)
        _data_id, value = self._references[refid]
        return value

    def unpack_unordered_values(self, values):
        """Unpack an unordered list of values taking DelayPacking
        exceptions into account to resolve circular references .
        Used to unpack set values when order is not guaranteed by
        the serializer. See unpack_unordered_pairs()."""

        values = list(values)  # To support iterators
        result = []

        # Try to unpack values more than one time to resolve cross references
        max_loop = 2
        while values and max_loop:
            next_values = []
            for value_data in values:
                blob = self._begin()
                try:
                    # try unpacking the value
                    value = self.unpack_data(value_data)
                    self._commit(blob)
                except DelayPacking:
                    self._rollback(blob)
                    # If it is delayed keep it for later
                    next_values.append(value_data)
                    continue
                result.append(value)
            values = next_values
            max_loop -= 1

        if values:
            # Not all items were resolved
            raise DelayPacking()

        return result

    def unpack_unordered_pairs(self, pairs):
        """Unpack an unordered list of value pairs taking DelayPacking
        exceptions into account to resolve circular references .
        Used to unpack dictionary items when the order is not guarennteed
        by the serializer. When item order change between packing
        and unpacking, references are not guaranteed to appear before
        dereferences anymore. So if unpacking an item fail because
        of unknown dereference, we must keep it aside, continue unpacking
        the other items and continue later."""

        items = [(False, k, v) for k, v in pairs]
        result = []

        # Try to unpack items more than one time to resolve cross references
        max_loop = 2
        while items and max_loop:
            next_items = []
            for key_unpacked, key_data, value_data in items:
                if key_unpacked:
                    key = key_data
                else:
                    blob = self._begin()
                    try:
                        # Try unpacking the key
                        key = self.unpack_data(key_data)
                        self._commit(blob)
                    except DelayPacking:
                        self._rollback(blob)
                        # If it is delayed keep it for later
                        next_items.append((False, key_data, value_data))
                        continue

                blob = self._begin()
                try:
                    # try unpacking the value
                    value = self.unpack_data(value_data)
                    self._commit(blob)
                except DelayPacking:
                    self._rollback(blob)
                    # If it is delayed keep it for later
                    next_items.append((True, key, value_data))
                    continue

                # Update the container with the unpacked value and key
                result.append((key, value))
            items = next_items
            max_loop -= 1

        if items:
            # Not all items were resolved
            raise DelayPacking()

        return result

    ### virtual ###

    def analyse_data(self, data):
        """Analyses the data provided and return a tuple containing
        the data type and a function to unpack it.
        The type can be None for immutable types, instances,
        reference and dereferences."""

    ### private ###

    def _begin(self):
        # Start a DelayPacking protected section
        blob = self._instances
        self._instances = []
        return blob

    def _rollback(self, blob):
        # We need to rollback after a DelayPacking has been raised
        # we only keep instances that has been referenced
        for instance in self._instances:
            refid = instance[3]
            if refid is not None:
                blob.append(instance)
        self._instances = blob

    def _commit(self, blob):
        # Commit after a DelayPacking protected section
        # Joining the instance lists
        blob.extend(self._instances)
        self._instances = blob

    def _lookup_restorator(self, type_name):
        # Lookup the registry for a IRestorator
        restorator = self._registry.lookup(type_name)
        if restorator is None:
            raise TypeError("Type %s not supported by unserializer %s"
                            % (type_name, reflect.canonical_name(self)))
        return restorator

    def _unpack_data(self, data, refid, refdata):
        # Just return pass-through types,
        # support sub-classed base types and metaclasses
        if set(type(data).__mro__) & self.pass_through_types:
            return data

        analysis = self.analyse_data(data)

        if analysis is not None:

            constructor, unpacker = analysis

            if constructor is None:
                # Immutable types
                return unpacker(self, data)

            if callable(constructor):
                # Unpack the mutable containers that provides constructor
                container = constructor()
                if container is not None:
                    if refid is not None:
                        self._references[refid] = (id(refdata), container)
                    return self.delayed_unpacking(container, unpacker,
                                                  self, container, data)

            else:
                # Instance type name
                prepared = self.prepare_instance(constructor)
                if prepared is None:
                    # Immutable instance
                    return unpacker(self, data, None, None, None)

                restorator, instance = prepared

                if refid is not None:
                    self._references[refid] = (id(refdata), instance)
                return self.delayed_unpacking(instance, unpacker, self, data,
                                              refid, restorator, instance)

        raise TypeError("Type %s not supported by unserializer %s"
                        % (type(data).__name__,
                           reflect.canonical_name(self)))

    def _continue_restoring_instance(self, restorator, instance, data, refid):
        snapshot = self.unpack_data(data)
        # Delay instance initialization to the end to be sure
        # all snapshots circular references have been resolved
        self._instances.append((restorator, instance, snapshot, refid))
        return instance

    def _adapt_snapshot(self, restorator, snapshot):
        try:
            adapter = IVersionAdapter(restorator)
        except TypeError:
            pass
        else:
            target = self.get_target_ver(restorator, snapshot)
            source = self.get_source_ver(restorator, snapshot)
            if target is not None and target != source:
                snapshot = adapter.adapt_version(snapshot, source, target)
        return snapshot

    def get_target_ver(self, restorator, snapshot):
        return self._target_ver

    def get_source_ver(self, restorator, snapshot):
        return self._source_ver
class Unserializer(object):
    """Base class for unserializers. It handle delayed unpacking
    and instance restoration to resolve circular references.
    If no registry instance is specified when created, the default
    global registry will be used.

    A pre-converter can be specified at creation time, if so the
    data will be first converted by the given converter and then
    the result will be unserialized. Used to parse data before
    unserializing.
    """

    pass_through_types = ()

    def __init__(self,
                 converter_caps=None,
                 pre_converter=None,
                 registry=None,
                 externalizer=None,
                 source_ver=None,
                 target_ver=None):
        global _global_registry
        assert ((source_ver is None) and (target_ver is None)) \
            or ((source_ver is not None) and (target_ver is not None))
        self.converter_capabilities = converter_caps or DEFAULT_CONVERTER_CAPS
        self._pre_converter = pre_converter and IConverter(pre_converter)
        self._registry = IRegistry(registry) if registry else _global_registry
        self._externalizer = externalizer and IExternalizer(externalizer)
        self._source_ver = source_ver
        self._target_ver = target_ver
        self.reset()

    ### IConverter ###

    def convert(self, data):
        try:
            # Pre-convert the data if a convertor was specified
            converted = self.pre_convertion(data)
            # Unpack the first level of values
            unpacked = self.unpack_data(converted)
            # Continue unpacking level by level
            self.finish_unpacking()
            # Inform object that it has migrated if this is a case
            if (IVersionAdapter.providedBy(unpacked) and self._migrated):
                unpacked.set_migrated()

            # Should be finished by now
            return unpacked
        finally:
            # Reset the state to cleanup all references
            self.reset()

    ### protected ###

    def pre_convertion(self, data):
        if self._pre_converter is not None:
            return self._pre_converter.convert(data)
        return data

    def reset(self):
        self._references = {}  # {REFERENCE_ID: (DATA_ID, OBJECT)}
        self._pending = []  # Pendings unpacking
        self._instances = []  # [(RESTORATOR, INSTANCE, SNAPSHOT, REFID)]
        self._delayed = 0  # If we are in a delayable unpacking
        # If some snapshot has been migrated between versions
        self._migrated = False

    def unpack_data(self, data):
        return self._unpack_data(data, None, None)

    def delayed_unpacking(self, container, fun, *args, **kwargs):
        """Should be used when unpacking mutable values.
        This allows circular references resolution by pausing serialization."""
        try:
            self._delayed += 1
            blob = self._begin()
            try:
                fun(*args, **kwargs)
                self._commit(blob)
                return container
            except DelayPacking:
                self._rollback(blob)
                continuation = (fun, args, kwargs)
                self._pending.append(continuation)
                return container
        finally:
            self._delayed -= 1

    def finish_unpacking(self):
        while self._pending:
            fun, args, kwargs = self._pending.pop(0)
            fun(*args, **kwargs)

        # Initialize delayed mutable instances in creation order
        for restorator, instance, snapshot, _refid in self._instances:
            if restorator is not None:
                # delayed mutable instances
                snapshot = self._adapt_snapshot(restorator, snapshot)
                instance.recover(snapshot)

        # Calls the instances post restoration callback in reversed order
        # in an intent to reduce the possibilities of instances relying
        # on there references being fully restored when called.
        # This should not be relied on anyway.
        for _, instance, _, _ in reversed(self._instances):
            restored_fun = getattr(instance, "restored", None)
            if restored_fun is not None:
                restored_fun()

    def restore_type(self, type_name):
        value = reflect.named_object(type_name)
        if issubclass(value, type):
            raise ValueError("type %r unserialized to something that "
                             "isn't a type: %r" % (type_name, value))
        return value

    def restore_external(self, data):
        if self._externalizer is None:
            raise ValueError("Got external reference %r but unserializer "
                             "do not have any IExternalizer")
        identifier = self.unpack_data(data)
        instance = self._externalizer.lookup(identifier)
        if instance is None:
            raise ValueError("No external reference found with identifier %r" %
                             (identifier, ))
        return instance

    def prepare_instance(self, type_name):
        restorator = self._lookup_restorator(type_name)
        # Prepare the instance for recovery
        instance = restorator.prepare()
        if instance is not None:
            return restorator, instance

    def restore_instance(self,
                         type_name,
                         data,
                         refid=None,
                         restorator=None,
                         instance=None):
        if restorator is None:
            restorator = self._lookup_restorator(type_name)

        if instance is None:
            # Prepare the instance for recovery
            instance = restorator.prepare()

        if instance is None:
            # Immutable type, we can't delay restoration
            snapshot = self.unpack_data(data)
            snapshot = self._adapt_snapshot(restorator, snapshot)
            instance = restorator.restore(snapshot)
            self._instances.append((None, instance, None, refid))

            return instance

        # Delay the instance restoration for later to handle circular refs
        return self.delayed_unpacking(instance,
                                      self._continue_restoring_instance,
                                      restorator, instance, data, refid)

    def restore_reference(self, refid, data):
        if refid in self._references:
            # This is because of DelayUnpacking exception, reference
            # can be registered multiple times
            data_id, value = self._references[refid]
            if data_id == id(data):
                return value
            raise ValueError("Multiple references found with "
                             "the same identifier: %s" % refid)
        value = self._unpack_data(data, refid, data)
        if refid not in self._references:
            # If not yet referenced
            self._references[refid] = (id(data), value)
        return value

    def restore_dereference(self, refid):
        if refid not in self._references:
            # Dereference to an unknown reference
            if self._delayed > 0:
                # If we unpacking can be delayed because we are unpacking
                # a mutable object just delay the unpacking for later
                raise DelayPacking()
            raise ValueError("Dereferencing of yet unknown reference: %s" %
                             refid)
        _data_id, value = self._references[refid]
        return value

    def unpack_unordered_values(self, values):
        """Unpack an unordered list of values taking DelayPacking
        exceptions into account to resolve circular references .
        Used to unpack set values when order is not guaranteed by
        the serializer. See unpack_unordered_pairs()."""

        values = list(values)  # To support iterators
        result = []

        # Try to unpack values more than one time to resolve cross references
        max_loop = 2
        while values and max_loop:
            next_values = []
            for value_data in values:
                blob = self._begin()
                try:
                    # try unpacking the value
                    value = self.unpack_data(value_data)
                    self._commit(blob)
                except DelayPacking:
                    self._rollback(blob)
                    # If it is delayed keep it for later
                    next_values.append(value_data)
                    continue
                result.append(value)
            values = next_values
            max_loop -= 1

        if values:
            # Not all items were resolved
            raise DelayPacking()

        return result

    def unpack_unordered_pairs(self, pairs):
        """Unpack an unordered list of value pairs taking DelayPacking
        exceptions into account to resolve circular references .
        Used to unpack dictionary items when the order is not guarennteed
        by the serializer. When item order change between packing
        and unpacking, references are not guaranteed to appear before
        dereferences anymore. So if unpacking an item fail because
        of unknown dereference, we must keep it aside, continue unpacking
        the other items and continue later."""

        items = [(False, k, v) for k, v in pairs]
        result = []

        # Try to unpack items more than one time to resolve cross references
        max_loop = 2
        while items and max_loop:
            next_items = []
            for key_unpacked, key_data, value_data in items:
                if key_unpacked:
                    key = key_data
                else:
                    blob = self._begin()
                    try:
                        # Try unpacking the key
                        key = self.unpack_data(key_data)
                        self._commit(blob)
                    except DelayPacking:
                        self._rollback(blob)
                        # If it is delayed keep it for later
                        next_items.append((False, key_data, value_data))
                        continue

                blob = self._begin()
                try:
                    # try unpacking the value
                    value = self.unpack_data(value_data)
                    self._commit(blob)
                except DelayPacking:
                    self._rollback(blob)
                    # If it is delayed keep it for later
                    next_items.append((True, key, value_data))
                    continue

                # Update the container with the unpacked value and key
                result.append((key, value))
            items = next_items
            max_loop -= 1

        if items:
            # Not all items were resolved
            raise DelayPacking()

        return result

    ### virtual ###

    def analyse_data(self, data):
        """Analyses the data provided and return a tuple containing
        the data type and a function to unpack it.
        The type can be None for immutable types, instances,
        reference and dereferences."""

    ### private ###

    def _begin(self):
        # Start a DelayPacking protected section
        blob = self._instances
        self._instances = []
        return blob

    def _rollback(self, blob):
        # We need to rollback after a DelayPacking has been raised
        # we only keep instances that has been referenced
        for instance in self._instances:
            refid = instance[3]
            if refid is not None:
                blob.append(instance)
        self._instances = blob

    def _commit(self, blob):
        # Commit after a DelayPacking protected section
        # Joining the instance lists
        blob.extend(self._instances)
        self._instances = blob

    def _lookup_restorator(self, type_name):
        # Lookup the registry for a IRestorator
        restorator = self._registry.lookup(type_name)
        if restorator is None:
            raise TypeError("Type %s not supported by unserializer %s" %
                            (type_name, reflect.canonical_name(self)))
        return restorator

    def _unpack_data(self, data, refid, refdata):
        # Just return pass-through types,
        # support sub-classed base types and metaclasses
        if set(type(data).__mro__) & self.pass_through_types:
            return data

        analysis = self.analyse_data(data)

        if analysis is not None:

            constructor, unpacker = analysis

            if constructor is None:
                # Immutable types
                return unpacker(self, data)

            if callable(constructor):
                # Unpack the mutable containers that provides constructor
                container = constructor()
                if container is not None:
                    if refid is not None:
                        self._references[refid] = (id(refdata), container)
                    return self.delayed_unpacking(container, unpacker, self,
                                                  container, data)

            else:
                # Instance type name
                prepared = self.prepare_instance(constructor)
                if prepared is None:
                    # Immutable instance
                    return unpacker(self, data, None, None, None)

                restorator, instance = prepared

                if refid is not None:
                    self._references[refid] = (id(refdata), instance)
                return self.delayed_unpacking(instance, unpacker, self, data,
                                              refid, restorator, instance)

        raise TypeError("Type %s not supported by unserializer %s" %
                        (type(data).__name__, reflect.canonical_name(self)))

    def _continue_restoring_instance(self, restorator, instance, data, refid):
        snapshot = self.unpack_data(data)
        # Delay instance initialization to the end to be sure
        # all snapshots circular references have been resolved
        self._instances.append((restorator, instance, snapshot, refid))
        return instance

    def _adapt_snapshot(self, restorator, snapshot):
        try:
            adapter = IVersionAdapter(restorator)
        except TypeError:
            pass
        else:
            target = self.get_target_ver(restorator, snapshot)
            source = self.get_source_ver(restorator, snapshot)
            if target is not None and target != source:
                snapshot = adapter.adapt_version(snapshot, source, target)
        return snapshot

    def get_target_ver(self, restorator, snapshot):
        return self._target_ver

    def get_source_ver(self, restorator, snapshot):
        return self._source_ver