Exemple #1
0
    def _restore_object_instance_variables(self, obj, instance):
        for k, v in sorted(obj.items(), key=util.itemgetter):
            # ignore the reserved attribute
            if k in tags.RESERVED:
                continue
            self._namestack.append(k)
            # step into the namespace
            value = self._restore(v)
            if (util.is_noncomplex(instance)
                    or util.is_dictionary_subclass(instance)):
                instance[k] = value
            else:
                setattr(instance, k, value)
            # step out
            self._namestack.pop()

        # Handle list and set subclasses
        if has_tag(obj, tags.SEQ):
            if hasattr(instance, 'append'):
                for v in obj[tags.SEQ]:
                    instance.append(self._restore(v))
            if hasattr(instance, 'add'):
                for v in obj[tags.SEQ]:
                    instance.add(self._restore(v))

        if _supports_getstate(obj, instance):
            self._restore_state(obj, instance)

        return instance
    def _flatten_obj_instance(self, obj):
        """Recursively flatten an instance and return a json-friendly dict
        """
        data = {}
        has_class = hasattr(obj, '__class__')
        has_dict = hasattr(obj, '__dict__')
        has_slots = not has_dict and hasattr(obj, '__slots__')
        has_getstate = has_dict and hasattr(obj, '__getstate__')
        has_getstate_support = has_getstate and hasattr(obj, '__setstate__')
        HandlerClass = handlers.registry.get(type(obj))

        if (has_class and not util.is_repr(obj) and
                not util.is_module(obj)):
            module, name = _getclassdetail(obj)
            if self.unpicklable is True:
                data[tags.OBJECT] = '%s.%s' % (module, name)
            # Check for a custom handler
            if HandlerClass:
                handler = HandlerClass(self)
                return handler.flatten(obj, data)

        if util.is_module(obj):
            if self.unpicklable is True:
                data[tags.REPR] = '%s/%s' % (obj.__name__,
                                             obj.__name__)
            else:
                data = unicode(obj)
            return data

        if util.is_repr(obj):
            if self.unpicklable is True:
                data[tags.REPR] = '%s/%s' % (obj.__class__.__module__,
                                             repr(obj))
            else:
                data = unicode(obj)
            return data

        if util.is_dictionary_subclass(obj):
            return self._flatten_dict_obj(obj, data)

        if util.is_noncomplex(obj):
            return [self.flatten(v) for v in obj]

        if has_dict:
            # Support objects that subclasses list and set
            if util.is_collection_subclass(obj):
                return self._flatten_collection_obj(obj, data)

            # Support objects with __getstate__(); this ensures that
            # both __setstate__() and __getstate__() are implemented
            if has_getstate_support:
                data[tags.STATE] = self.flatten(obj.__getstate__())
                return data

            # hack for zope persistent objects; this unghostifies the object
            getattr(obj, '_', None)
            return self._flatten_dict_obj(obj.__dict__, data)

        if has_slots:
            return self._flatten_newstyle_with_slots(obj, data)
Exemple #3
0
    def _restore_from_dict(self, obj, cls, instance, ignorereserved=True):
        restore_key = self._restore_key_fn()
        method = _obj_setattr

        for k in util.get_public_variables(cls):
            if k in obj:
                v = obj[k]
                # ignore the reserved attribute
                if ignorereserved and k in tags.RESERVED:
                    continue
                if isinstance(k, numeric_types):
                    str_k = unicode(k)
                else:
                    str_k = k
                self._namestack.append(str_k)
                k = restore_key(k)
                # step into the namespace
                value = self._restore(v)
                if (util.is_noncomplex(instance)
                        or util.is_dictionary_subclass(instance)):
                    instance[k] = value
                else:
                    setattr(instance, k, value)

                # This instance has an instance variable named `k` that is
                # currently a proxy and must be replaced
                if isinstance(value, _Proxy):
                    self._proxies.append((instance, k, value, method))

                # step out
                self._namestack.pop()
            else:
                # Attribute in cls but not given in JSON. Assign it to
                # None so that user could tell that it wasn't given.
                setattr(instance, k, None)
    def _restore_from_dict(self, obj, instance, ignorereserved=True):
        restore_key = self._restore_key_fn()
        method = _obj_setattr

        for k, v in sorted(obj.items(), key=util.itemgetter):
            # ignore the reserved attribute
            if ignorereserved and k in tags.RESERVED:
                continue
            self._namestack.append(k)
            k = restore_key(k)
            # step into the namespace
            value = self._restore(v)
            if (util.is_noncomplex(instance) or
                    util.is_dictionary_subclass(instance)):
                instance[k] = value
            else:
                setattr(instance, k, value)

            # This instance has an instance variable named `k` that is
            # currently a proxy and must be replaced
            if type(value) is _Proxy:
                self._proxies.append((instance, k, value, method))

            # step out
            self._namestack.pop()
    def _restore_object_instance_variables(self, obj, instance):
        if hasattr(instance, '__setstate__') and has_tag(obj, tags.STATE):
            state = self._restore(obj[tags.STATE])
            instance.__setstate__(state)
            return instance

        for k, v in sorted(obj.items(), key=operator.itemgetter(0)):
            # ignore the reserved attribute
            if k in tags.RESERVED:
                continue
            self._namestack.append(k)
            # step into the namespace
            value = self._restore(v)
            if (util.is_noncomplex(instance) or
                    util.is_dictionary_subclass(instance)):
                instance[k] = value
            else:
                setattr(instance, k, value)
            # step out
            self._namestack.pop()

        # Handle list and set subclasses
        if has_tag(obj, tags.SEQ):
            if hasattr(instance, 'append'):
                for v in obj[tags.SEQ]:
                    instance.append(self._restore(v))
            if hasattr(instance, 'add'):
                for v in obj[tags.SEQ]:
                    instance.add(self._restore(v))

        return instance
Exemple #6
0
    def _restore_object_instance_variables(self, obj, instance):
        for k, v in sorted(obj.items(), key=util.itemgetter):
            # ignore the reserved attribute
            if k in tags.RESERVED:
                continue
            self._namestack.append(k)
            # step into the namespace
            value = self._restore(v)
            if util.is_noncomplex(instance) or util.is_dictionary_subclass(instance):
                instance[k] = value
            else:
                setattr(instance, k, value)
            # step out
            self._namestack.pop()

        # Handle list and set subclasses
        if has_tag(obj, tags.SEQ):
            if hasattr(instance, "append"):
                for v in obj[tags.SEQ]:
                    instance.append(self._restore(v))
            if hasattr(instance, "add"):
                for v in obj[tags.SEQ]:
                    instance.add(self._restore(v))

        if _supports_getstate(obj, instance):
            self._restore_state(obj, instance)

        return instance
Exemple #7
0
    def _flatten_obj_instance(self, obj):
        """Recursively flatten an instance and return a json-friendly dict
        """
        data = {}
        has_class = hasattr(obj, '__class__')
        has_dict = hasattr(obj, '__dict__')
        has_slots = not has_dict and hasattr(obj, '__slots__')
        has_getstate = has_dict and hasattr(obj, '__getstate__')
        has_getstate_support = has_getstate and hasattr(obj, '__setstate__')

        HandlerClass = handlers.registry.get(type(obj))

        if has_class and not util.is_module(obj):
            module, name = _getclassdetail(obj)
            if self.unpicklable:
                data[tags.OBJECT] = '%s.%s' % (module, name)
            # Check for a custom handler
            if HandlerClass:
                handler = HandlerClass(self)
                flat_obj = handler.flatten(obj, data)
                self._mkref(flat_obj)
                return flat_obj

        if util.is_module(obj):
            if self.unpicklable:
                data[tags.REPR] = '%s/%s' % (obj.__name__, obj.__name__)
            else:
                data = unicode(obj)
            return data

        if util.is_dictionary_subclass(obj):
            return self._flatten_dict_obj(obj, data)

        if has_dict:
            # Support objects that subclasses list and set
            if util.is_collection_subclass(obj):
                return self._flatten_collection_obj(obj, data)

            # Support objects with __getstate__(); this ensures that
            # both __setstate__() and __getstate__() are implemented
            if has_getstate_support:
                state = self.flatten(obj.__getstate__())
                if self.unpicklable:
                    data[tags.STATE] = state
                else:
                    data = state
                return data

            # hack for zope persistent objects; this unghostifies the object
            getattr(obj, '_', None)
            return self._flatten_dict_obj(obj.__dict__, data)

        if util.is_collection_subclass(obj):
            return self._flatten_collection_obj(obj, data)

        if util.is_noncomplex(obj):
            return [self.flatten(v) for v in obj]

        if has_slots:
            return self._flatten_newstyle_with_slots(obj, data)
Exemple #8
0
    def _restore_from_dict(self, obj, instance, ignorereserved=True):
        restore_key = self._restore_key_fn()
        method = _obj_setattr

        for k, v in sorted(obj.items(), key=util.itemgetter):
            # ignore the reserved attribute
            if ignorereserved and k in tags.RESERVED:
                continue
            if isinstance(k, numeric_types):
                str_k = unicode(k)
            else:
                str_k = k
            self._namestack.append(str_k)
            k = restore_key(k)
            # step into the namespace
            value = self._restore(v)
            if (util.is_noncomplex(instance)
                    or util.is_dictionary_subclass(instance)):
                instance[k] = value
            else:
                setattr(instance, k, value)

            # This instance has an instance variable named `k` that is
            # currently a proxy and must be replaced
            if isinstance(value, _Proxy):
                self._proxies.append((instance, k, value, method))

            # step out
            self._namestack.pop()
Exemple #9
0
    def restore(self, obj):
        """Restores a flattened object to its original python state.

        Simply returns any of the basic builtin types

        >>> u = Unpickler()
        >>> u.restore('hello world')
        'hello world'
        >>> u.restore({'key': 'value'})
        {'key': 'value'}
        """
        self._push()

        if has_tag(obj, tags.ID):
            return self._pop(self._objs[obj[tags.ID]])

        # Backwards compatibility
        if has_tag(obj, tags.REF):
            return self._pop(self._namedict.get(obj[tags.REF]))

        if has_tag(obj, tags.TYPE):
            typeref = loadclass(obj[tags.TYPE])
            if not typeref:
                return self._pop(obj)
            return self._pop(typeref)

        # Backwards compatibility
        if has_tag(obj, tags.REPR):
            obj = loadrepr(obj[tags.REPR])
            return self._pop(self._mkref(obj))

        if has_tag(obj, tags.OBJECT):
            cls = loadclass(obj[tags.OBJECT])
            if not cls:
                return self._pop(self._mkref(obj))

            # check custom handlers
            HandlerClass = handlers.BaseHandler._registry.get(cls)
            if HandlerClass:
                handler = HandlerClass(self)
                instance = handler.restore(obj)
                return self._pop(self._mkref(instance))

            factory = loadfactory(obj)
            args = getargs(obj)
            if args:
                args = self.restore(args)
            try:
                if hasattr(cls, '__new__'):
                    # new style classes
                    if factory:
                        instance = cls.__new__(cls, factory, *args)
                        instance.default_factory = factory
                    else:
                        instance = cls.__new__(cls, *args)
                else:
                    instance = object.__new__(cls)
            except TypeError:
                # old-style classes
                try:
                    instance = cls()
                except TypeError:
                    # fail gracefully if the constructor requires arguments
                    return self._pop(self._mkref(obj))

            # Add to the instance table to allow being referenced by a
            # downstream object
            self._mkref(instance)

            if isinstance(instance, tuple):
                return self._pop(instance)

            if hasattr(instance, '__setstate__') and has_tag(obj, tags.STATE):
                state = self.restore(obj[tags.STATE])
                instance.__setstate__(state)
                return self._pop(instance)

            for k, v in sorted(obj.items(), key=operator.itemgetter(0)):
                # ignore the reserved attribute
                if k in tags.RESERVED:
                    continue
                self._namestack.append(k)
                # step into the namespace
                value = self.restore(v)
                if (util.is_noncomplex(instance)
                        or util.is_dictionary_subclass(instance)):
                    instance[k] = value
                else:
                    setattr(instance, k, value)
                # step out
                self._namestack.pop()

            # Handle list and set subclasses
            if has_tag(obj, tags.SEQ):
                if hasattr(instance, 'append'):
                    for v in obj[tags.SEQ]:
                        instance.append(self.restore(v))
                if hasattr(instance, 'add'):
                    for v in obj[tags.SEQ]:
                        instance.add(self.restore(v))

            return self._pop(instance)

        if util.is_list(obj):
            parent = []
            self._mkref(parent)
            children = [self.restore(v) for v in obj]
            parent.extend(children)
            return self._pop(parent)

        if has_tag(obj, tags.TUPLE):
            return self._pop(tuple([self.restore(v) for v in obj[tags.TUPLE]]))

        if has_tag(obj, tags.SET):
            return self._pop(set([self.restore(v) for v in obj[tags.SET]]))

        if util.is_dictionary(obj):
            data = {}
            for k, v in sorted(obj.items(), key=operator.itemgetter(0)):
                self._namestack.append(k)
                data[k] = self.restore(v)
                self._namestack.pop()

            return self._pop(data)

        return self._pop(obj)
Exemple #10
0
    def restore(self, obj):
        """Restores a flattened object to its original python state.

        Simply returns any of the basic builtin types

        >>> u = Unpickler()
        >>> u.restore('hello world')
        'hello world'
        >>> u.restore({'key': 'value'})
        {'key': 'value'}
        """
        self._push()

        if has_tag(obj, tags.ID):
            return self._pop(self._objs[obj[tags.ID]])

        if has_tag(obj, tags.REF):
            return self._pop(self._namedict.get(obj[tags.REF]))

        if has_tag(obj, tags.TYPE):
            typeref = loadclass(obj[tags.TYPE])
            if not typeref:
                return self._pop(obj)
            return self._pop(typeref)

        if has_tag(obj, tags.REPR):
            return self._pop(loadrepr(obj[tags.REPR]))

        if has_tag(obj, tags.OBJECT):

            cls = loadclass(obj[tags.OBJECT])
            if not cls:
                return self._pop(obj)

            # check custom handlers
            HandlerClass = handlers.registry.get(cls)
            if HandlerClass:
                handler = HandlerClass(self)
                return self._pop(handler.restore(obj))

            try:
                if hasattr(cls, '__new__'):
                    instance = cls.__new__(cls)
                else:
                    instance = object.__new__(cls)
            except TypeError:
                # old-style classes
                try:
                    instance = cls()
                except TypeError:
                    # fail gracefully if the constructor requires arguments
                    self._mkref(obj)
                    return self._pop(obj)

            # keep a obj->name mapping for use in the _isobjref() case
            self._mkref(instance)

            if hasattr(instance, '__setstate__') and has_tag(obj, tags.STATE):
                state = self.restore(obj[tags.STATE])
                instance.__setstate__(state)
                return self._pop(instance)

            for k, v in sorted(obj.iteritems(), key=operator.itemgetter(0)):
                # ignore the reserved attribute
                if k in tags.RESERVED:
                    continue
                self._namestack.append(k)
                # step into the namespace
                value = self.restore(v)
                if (util.is_noncomplex(instance) or
                        util.is_dictionary_subclass(instance)):
                    instance[k] = value
                else:
                    setattr(instance, k, value)
                # step out
                self._namestack.pop()

            # Handle list and set subclasses
            if has_tag(obj, tags.SEQ):
                if hasattr(instance, 'append'):
                    for v in obj[tags.SEQ]:
                        instance.append(self.restore(v))
                if hasattr(instance, 'add'):
                    for v in obj[tags.SEQ]:
                        instance.add(self.restore(v))

            return self._pop(instance)

        if util.is_list(obj):
            return self._pop([self.restore(v) for v in obj])

        if has_tag(obj, tags.TUPLE):
            return self._pop(tuple([self.restore(v) for v in obj[tags.TUPLE]]))

        if has_tag(obj, tags.SET):
            return self._pop(set([self.restore(v) for v in obj[tags.SET]]))

        if util.is_dictionary(obj):
            data = {}
            for k, v in sorted(obj.iteritems(), key=operator.itemgetter(0)):
                self._namestack.append(k)
                data[k] = self.restore(v)
                self._namestack.pop()
            return self._pop(data)

        return self._pop(obj)
Exemple #11
0
    def _flatten_obj_instance(self, obj):
        """Recursively flatten an instance and return a json-friendly dict
        """
        data = {}
        has_class = hasattr(obj, '__class__')
        has_dict = hasattr(obj, '__dict__')
        has_slots = not has_dict and hasattr(obj, '__slots__')
        has_getnewargs = hasattr(obj, '__getnewargs__')
        has_getnewargs_ex = hasattr(obj, '__getnewargs_ex__')
        has_getinitargs = hasattr(obj, '__getinitargs__')
        has_reduce, has_reduce_ex = util.has_reduce(obj)

        # Support objects with __getstate__(); this ensures that
        # both __setstate__() and __getstate__() are implemented
        has_getstate = hasattr(obj, '__getstate__')

        if has_class:
            cls = obj.__class__
        else:
            cls = type(obj)

        # Check for a custom handler
        class_name = util.importable_name(cls)
        handler = handlers.get(class_name)
        if handler is not None:
            if self.unpicklable:
                data[tags.OBJECT] = class_name
            return handler(self).flatten(obj, data)

        reduce_val = None
        if has_class and not util.is_module(obj):
            if self.unpicklable:
                class_name = util.importable_name(cls)
                data[tags.OBJECT] = class_name

            # test for a reduce implementation, and redirect before doing anything else
            # if that is what reduce requests
            if has_reduce_ex:
                try:
                    # we're implementing protocol 2
                    reduce_val = obj.__reduce_ex__(2)
                except TypeError:
                    # A lot of builtin types have a reduce which just raises a TypeError
                    # we ignore those
                    pass

            if has_reduce and not reduce_val:
                try:
                    reduce_val = obj.__reduce__()
                except TypeError:
                    # A lot of builtin types have a reduce which just raises a TypeError
                    # we ignore those
                    pass

            if reduce_val:
                try:
                    # At this stage, we only handle the case where __reduce__ returns a string
                    # other reduce functionality is implemented further down
                    if isinstance(reduce_val, (str, unicode)):
                        varpath = iter(reduce_val.split('.'))
                        # curmod will be transformed by the loop into the value to pickle
                        curmod = sys.modules[next(varpath)]
                        for modname in varpath:
                            curmod = getattr(curmod, modname)
                            # replace obj with value retrieved
                        return self._flatten(curmod)
                except KeyError:
                    # well, we can't do anything with that, so we ignore it
                    pass

            if has_getnewargs_ex:
                data[tags.NEWARGSEX] = list(
                    map(self._flatten, obj.__getnewargs_ex__()))

            if has_getnewargs and not has_getnewargs_ex:
                data[tags.NEWARGS] = self._flatten(obj.__getnewargs__())

            if has_getinitargs:
                data[tags.INITARGS] = self._flatten(obj.__getinitargs__())

        if has_getstate:
            try:
                state = obj.__getstate__()
            except TypeError:
                # Has getstate but it cannot be called, e.g. file descriptors
                # in Python3
                self._pickle_warning(obj)
                return None
            else:
                return self._getstate(state, data)

        if util.is_module(obj):
            if self.unpicklable:
                data[tags.REPR] = '%s/%s' % (obj.__name__, obj.__name__)
            else:
                data = unicode(obj)
            return data

        if util.is_dictionary_subclass(obj):
            self._flatten_dict_obj(obj, data)
            return data

        if util.is_sequence_subclass(obj):
            return self._flatten_sequence_obj(obj, data)

        if util.is_noncomplex(obj):
            return [self._flatten(v) for v in obj]

        if util.is_iterator(obj):
            # force list in python 3
            data[tags.ITERATOR] = list(
                map(self._flatten, islice(obj, self._max_iter)))
            return data

        if reduce_val and not isinstance(reduce_val, (str, unicode)):
            # at this point, reduce_val should be some kind of iterable
            # pad out to len 5
            rv_as_list = list(reduce_val)
            insufficiency = 5 - len(rv_as_list)
            if insufficiency:
                rv_as_list += [None] * insufficiency

            if rv_as_list[0].__name__ == '__newobj__':
                rv_as_list[0] = tags.NEWOBJ

            data[tags.REDUCE] = list(map(self._flatten, rv_as_list))

            # lift out iterators, so we don't have to iterator and uniterator their content
            # on unpickle
            if data[tags.REDUCE][3]:
                data[tags.REDUCE][3] = data[tags.REDUCE][3][tags.ITERATOR]

            if data[tags.REDUCE][4]:
                data[tags.REDUCE][4] = data[tags.REDUCE][4][tags.ITERATOR]

            return data

        if has_dict:
            # Support objects that subclasses list and set
            if util.is_sequence_subclass(obj):
                return self._flatten_sequence_obj(obj, data)

            # hack for zope persistent objects; this unghostifies the object
            getattr(obj, '_', None)
            return self._flatten_dict_obj(obj.__dict__, data)

        if has_slots:
            return self._flatten_newstyle_with_slots(obj, data)

        self._pickle_warning(obj)
        return None
Exemple #12
0
 def test_is_dictionary_subclass_subclass(self):
     self.assertTrue(is_dictionary_subclass(DictSubclass()))
Exemple #13
0
 def test_is_dictionary_subclass_dict(self):
     self.assertFalse(is_dictionary_subclass({}))
Exemple #14
0
    def _flatten_obj_instance(self, obj):
        """Recursively flatten an instance and return a json-friendly dict
        """
        data = {}
        has_class = hasattr(obj, '__class__')
        has_dict = hasattr(obj, '__dict__')
        has_slots = not has_dict and hasattr(obj, '__slots__')
        has_getnewargs = util.has_method(obj, '__getnewargs__')
        has_getnewargs_ex = util.has_method(obj, '__getnewargs_ex__')
        has_getinitargs = util.has_method(obj, '__getinitargs__')
        has_reduce, has_reduce_ex = util.has_reduce(obj)

        # Support objects with __getstate__(); this ensures that
        # both __setstate__() and __getstate__() are implemented
        has_getstate = hasattr(obj, '__getstate__')
        # not using has_method since __getstate__() is handled separately below

        if has_class:
            cls = obj.__class__
        else:
            cls = type(obj)

        # Check for a custom handler
        class_name = util.importable_name(cls)
        handler = handlers.get(cls, handlers.get(class_name))
        if handler is not None:
            if self.unpicklable:
                data[tags.OBJECT] = class_name
            return handler(self).flatten(obj, data)

        reduce_val = None
        if has_class and not util.is_module(obj):
            if self.unpicklable:
                class_name = util.importable_name(cls)
                data[tags.OBJECT] = class_name

            # test for a reduce implementation, and redirect before doing anything else
            # if that is what reduce requests
            if has_reduce_ex:
                try:
                    # we're implementing protocol 2
                    reduce_val = obj.__reduce_ex__(2)
                except TypeError:
                    # A lot of builtin types have a reduce which just raises a TypeError
                    # we ignore those
                    pass

            if has_reduce and not reduce_val:
                try:
                    reduce_val = obj.__reduce__()
                except TypeError:
                    # A lot of builtin types have a reduce which just raises a TypeError
                    # we ignore those
                    pass

            if reduce_val:
                try:
                    # At this stage, we only handle the case where __reduce__ returns a string
                    # other reduce functionality is implemented further down
                    if isinstance(reduce_val, (str, unicode)):
                        varpath = iter(reduce_val.split('.'))
                        # curmod will be transformed by the loop into the value to pickle
                        curmod = sys.modules[next(varpath)]
                        for modname in varpath:
                            curmod = getattr(curmod, modname)
                            # replace obj with value retrieved
                        return self._flatten(curmod)
                except KeyError:
                    # well, we can't do anything with that, so we ignore it
                    pass

            if has_getnewargs_ex:
                data[tags.NEWARGSEX] = list(map(self._flatten, obj.__getnewargs_ex__()))

            if has_getnewargs and not has_getnewargs_ex:
                data[tags.NEWARGS] = self._flatten(obj.__getnewargs__())

            if has_getinitargs:
                data[tags.INITARGS] = self._flatten(obj.__getinitargs__())

        if has_getstate:
            try:
                state = obj.__getstate__()
            except TypeError:
                # Has getstate but it cannot be called, e.g. file descriptors
                # in Python3
                self._pickle_warning(obj)
                return None
            else:
                return self._getstate(state, data)

        if util.is_module(obj):
            if self.unpicklable:
                data[tags.REPR] = '%s/%s' % (obj.__name__,
                                             obj.__name__)
            else:
                data = unicode(obj)
            return data

        if util.is_dictionary_subclass(obj):
            self._flatten_dict_obj(obj, data)
            return data

        if util.is_sequence_subclass(obj):
            return self._flatten_sequence_obj(obj, data)

        if util.is_noncomplex(obj):
            return [self._flatten(v) for v in obj]

        if util.is_iterator(obj):
            # force list in python 3
            data[tags.ITERATOR] = list(map(self._flatten, islice(obj, self._max_iter)))
            return data

        if reduce_val and not isinstance(reduce_val, (str, unicode)):
            # at this point, reduce_val should be some kind of iterable
            # pad out to len 5
            rv_as_list = list(reduce_val)
            insufficiency = 5 - len(rv_as_list)
            if insufficiency:
                rv_as_list += [None] * insufficiency

            if rv_as_list[0].__name__ == '__newobj__':
                rv_as_list[0] = tags.NEWOBJ

            data[tags.REDUCE] = list(map(self._flatten, rv_as_list))

            # lift out iterators, so we don't have to iterator and uniterator their content
            # on unpickle
            if data[tags.REDUCE][3]:
                data[tags.REDUCE][3] = data[tags.REDUCE][3][tags.ITERATOR]

            if data[tags.REDUCE][4]:
                data[tags.REDUCE][4] = data[tags.REDUCE][4][tags.ITERATOR]

            return data

        if has_dict:
            # Support objects that subclasses list and set
            if util.is_sequence_subclass(obj):
                return self._flatten_sequence_obj(obj, data)

            # hack for zope persistent objects; this unghostifies the object
            getattr(obj, '_', None)
            return self._flatten_dict_obj(obj.__dict__, data)

        if has_slots:
            return self._flatten_newstyle_with_slots(obj, data)

        # catchall return for data created above without a return
        # (e.g. __getnewargs__ is not supposed to be the end of the story)
        if data:
            return data

        self._pickle_warning(obj)
        return None
Exemple #15
0
    def flatten(self, obj):
        """Takes an object and returns a JSON-safe representation of it.
        
        Simply returns any of the basic builtin datatypes
        
        >>> p = Pickler()
        >>> p.flatten('hello world')
        'hello world'
        >>> p.flatten(u'hello world')
        u'hello world'
        >>> p.flatten(49)
        49
        >>> p.flatten(350.0)
        350.0
        >>> p.flatten(True)
        True
        >>> p.flatten(False)
        False
        >>> r = p.flatten(None)
        >>> r is None
        True
        >>> p.flatten(False)
        False
        >>> p.flatten([1, 2, 3, 4])
        [1, 2, 3, 4]
        >>> p.flatten((1,2,))[tags.TUPLE]
        [1, 2]
        >>> p.flatten({'key': 'value'})
        {'key': 'value'}
        """

        self._push()

        if self._depth == self._max_depth:
            return self._pop(repr(obj))

        if util.is_primitive(obj):
            return self._pop(obj)

        if util.is_list(obj):
            return self._pop([self.flatten(v) for v in obj])

        # We handle tuples and sets by encoding them in a "(tuple|set)dict"
        if util.is_tuple(obj):
            return self._pop({tags.TUPLE: [self.flatten(v) for v in obj]})

        if util.is_set(obj):
            return self._pop({tags.SET: [self.flatten(v) for v in obj]})

        if util.is_dictionary(obj):
            return self._pop(self._flatten_dict_obj(obj, obj.__class__()))

        if util.is_type(obj):
            return self._pop(_mktyperef(obj))

        if util.is_object(obj):
            data = {}
            has_class = hasattr(obj, '__class__')
            has_dict = hasattr(obj, '__dict__')
            if self._mkref(obj):
                if has_class and not util.is_repr(obj):
                    module, name = _getclassdetail(obj)
                    if self.unpicklable is True:
                        data[tags.OBJECT] = '%s.%s' % (module, name)

                if util.is_repr(obj):
                    if self.unpicklable is True:
                        data[tags.REPR] = '%s/%s' % (obj.__class__.__module__,
                                                     repr(obj))
                    else:
                        data = unicode(obj)
                    return self._pop(data)

                if util.is_dictionary_subclass(obj):
                    return self._pop(self._flatten_dict_obj(obj, data))

                if util.is_noncomplex(obj):
                    return self._pop([self.flatten(v) for v in obj])

                if has_dict:
                    return self._pop(self._flatten_dict_obj(
                        obj.__dict__, data))
            else:
                # We've seen this object before so place an object
                # reference tag in the data. This avoids infinite recursion
                # when processing cyclical objects.
                return self._pop(self._getref(obj))

            return self._pop(data)
Exemple #16
0
    def flatten(self, obj):
        """Takes an object and returns a JSON-safe representation of it.
        
        Simply returns any of the basic builtin datatypes
        
        >>> p = Pickler()
        >>> p.flatten('hello world')
        'hello world'
        >>> p.flatten(u'hello world')
        u'hello world'
        >>> p.flatten(49)
        49
        >>> p.flatten(350.0)
        350.0
        >>> p.flatten(True)
        True
        >>> p.flatten(False)
        False
        >>> r = p.flatten(None)
        >>> r is None
        True
        >>> p.flatten(False)
        False
        >>> p.flatten([1, 2, 3, 4])
        [1, 2, 3, 4]
        >>> p.flatten((1,2,))[tags.TUPLE]
        [1, 2]
        >>> p.flatten({'key': 'value'})
        {'key': 'value'}
        """

        self._push()
        
        if self._depth == self._max_depth:
            return self._pop(repr(obj))

        if util.is_primitive(obj):
            return self._pop(obj)

        if util.is_list(obj):
            return self._pop([ self.flatten(v) for v in obj ])

        # We handle tuples and sets by encoding them in a "(tuple|set)dict"
        if util.is_tuple(obj):
            return self._pop({tags.TUPLE: [ self.flatten(v) for v in obj ]})

        if util.is_set(obj):
            return self._pop({tags.SET: [ self.flatten(v) for v in obj ]})

        if util.is_dictionary(obj):
            return self._pop(self._flatten_dict_obj(obj, obj.__class__()))

        if util.is_type(obj):
            return self._pop(_mktyperef(obj))

        if util.is_object(obj):
            data = {}
            has_class = hasattr(obj, '__class__')
            has_dict = hasattr(obj, '__dict__')
            if self._mkref(obj):
                if has_class and not util.is_repr(obj):
                    module, name = _getclassdetail(obj)
                    if self.unpicklable is True:
                        data[tags.OBJECT] = '%s.%s' % (module, name)

                if util.is_repr(obj):
                    if self.unpicklable is True:
                        data[tags.REPR] = '%s/%s' % (obj.__class__.__module__,
                                                     repr(obj))
                    else:
                        data = unicode(obj)
                    return self._pop(data)

                if util.is_dictionary_subclass(obj):
                    return self._pop(self._flatten_dict_obj(obj, data))

                if util.is_noncomplex(obj):
                    return self._pop([self.flatten(v) for v in obj])

                if has_dict:
                    return self._pop(self._flatten_dict_obj(obj.__dict__, data))
            else:
                # We've seen this object before so place an object
                # reference tag in the data. This avoids infinite recursion
                # when processing cyclical objects.
                return self._pop(self._getref(obj))

            return self._pop(data)
Exemple #17
0
    def restore(self, obj):
        """Restores a flattened object to its original python state.

        Simply returns any of the basic builtin types

        >>> u = Unpickler()
        >>> u.restore('hello world')
        'hello world'
        >>> u.restore({'key': 'value'})
        {'key': 'value'}
        """
        self._push()

        if has_tag(obj, tags.ID):
            return self._pop(self._objs[obj[tags.ID]])

        # Backwards compatibility
        if has_tag(obj, tags.REF):
            return self._pop(self._namedict.get(obj[tags.REF]))

        if has_tag(obj, tags.TYPE):
            typeref = loadclass(obj[tags.TYPE])
            if not typeref:
                return self._pop(obj)
            return self._pop(typeref)

        # Backwards compatibility
        if has_tag(obj, tags.REPR):
            obj = loadrepr(obj[tags.REPR])
            return self._pop(self._mkref(obj))

        if has_tag(obj, tags.OBJECT):
            cls = loadclass(obj[tags.OBJECT])
            if not cls:
                return self._pop(self._mkref(obj))

            # check custom handlers
            HandlerClass = handlers.registry.get(cls)
            if HandlerClass:
                handler = HandlerClass(self)
                instance = handler.restore(obj)
                return self._pop(self._mkref(instance))

            factory = loadfactory(obj)
            args = getargs(obj)
            if args:
                args = self.restore(args)
            try:
                if hasattr(cls, '__new__'):
                    # new style classes
                    if factory:
                        instance = cls.__new__(cls, factory, *args)
                        instance.default_factory = factory
                    else:
                        instance = cls.__new__(cls, *args)
                else:
                    instance = object.__new__(cls)
            except TypeError:
                # old-style classes
                try:
                    instance = cls()
                except TypeError:
                    # fail gracefully if the constructor requires arguments
                    return self._pop(self._mkref(obj))

            # Add to the instance table to allow being referenced by a
            # downstream object
            self._mkref(instance)

            if hasattr(instance, '__setstate__') and has_tag(obj, tags.STATE):
                state = self.restore(obj[tags.STATE])
                instance.__setstate__(state)
                return self._pop(instance)

            for k, v in sorted(obj.items(), key=operator.itemgetter(0)):
                # ignore the reserved attribute
                if k in tags.RESERVED:
                    continue
                self._namestack.append(k)
                # step into the namespace
                value = self.restore(v)
                if (util.is_noncomplex(instance) or
                        util.is_dictionary_subclass(instance)):
                    instance[k] = value
                else:
                    setattr(instance, k, value)
                # step out
                self._namestack.pop()

            # Handle list and set subclasses
            if has_tag(obj, tags.SEQ):
                if hasattr(instance, 'append'):
                    for v in obj[tags.SEQ]:
                        instance.append(self.restore(v))
                if hasattr(instance, 'add'):
                    for v in obj[tags.SEQ]:
                        instance.add(self.restore(v))

            return self._pop(instance)

        if util.is_list(obj):
            parent = []
            self._mkref(parent)
            children = [self.restore(v) for v in obj]
            parent.extend(children)
            return self._pop(parent)

        if has_tag(obj, tags.TUPLE):
            return self._pop(tuple([self.restore(v)
                                    for v in obj[tags.TUPLE]]))

        if has_tag(obj, tags.SET):
            return self._pop(set([self.restore(v)
                                  for v in obj[tags.SET]]))

        if util.is_dictionary(obj):
            data = {}
            for k, v in sorted(obj.items(), key=operator.itemgetter(0)):
                self._namestack.append(k)
                data[k] = self.restore(v)
                self._namestack.pop()

            return self._pop(data)

        return self._pop(obj)
Exemple #18
0
 def test_is_dictionary_subclass_subclass(self):
     self.assertTrue(util.is_dictionary_subclass(DictSubclass()))
Exemple #19
0
 def test_is_dictionary_subclass_dict(self):
     self.assertFalse(util.is_dictionary_subclass({}))
    def restore(self, obj):
        """Restores a flattened object to its original python state.
        
        Simply returns any of the basic builtin types
        
        >>> u = Unpickler()
        >>> u.restore('hello world')
        'hello world'
        >>> u.restore({'key': 'value'})
        {'key': 'value'}
        """
        self._push()

        if has_tag(obj, tags.REF):
            return self._pop(self._namedict.get(obj[tags.REF]))

        if has_tag(obj, tags.TYPE):
            typeref = loadclass(obj[tags.TYPE])
            if not typeref:
                return self._pop(obj)
            return self._pop(typeref)

        if has_tag(obj, tags.REPR):
            return self._pop(loadrepr(obj[tags.REPR]))

        if has_tag(obj, tags.OBJECT):

            cls = loadclass(obj[tags.OBJECT])
            if not cls:
                return self._pop(obj)
            try:
                instance = object.__new__(cls)
            except TypeError:
                # old-style classes
                try:
                    instance = cls()
                except TypeError:
                    # fail gracefully if the constructor requires arguments
                    self._mkref(obj)
                    return self._pop(obj)

            # keep a obj->name mapping for use in the _isobjref() case
            self._mkref(instance)

            for k, v in obj.iteritems():
                # ignore the reserved attribute
                if k in tags.RESERVED:
                    continue
                self._namestack.append(k)
                # step into the namespace
                value = self.restore(v)
                if util.is_noncomplex(instance) or util.is_dictionary_subclass(instance):
                    instance[k] = value
                else:
                    instance.__dict__[k] = value
                # step out
                self._namestack.pop()
            return self._pop(instance)

        if util.is_list(obj):
            return self._pop([self.restore(v) for v in obj])

        if has_tag(obj, tags.TUPLE):
            return self._pop(tuple([self.restore(v) for v in obj[tags.TUPLE]]))

        if has_tag(obj, tags.SET):
            return self._pop(set([self.restore(v) for v in obj[tags.SET]]))

        if util.is_dictionary(obj):
            data = {}
            for k, v in obj.iteritems():
                self._namestack.append(k)
                data[k] = self.restore(v)
                self._namestack.pop()
            return self._pop(data)

        return self._pop(obj)
Exemple #21
0
    def restore(self, obj):
        """Restores a flattened object to its original python state.
        
        Simply returns any of the basic builtin types
        
        >>> u = Unpickler()
        >>> u.restore('hello world')
        'hello world'
        >>> u.restore({'key': 'value'})
        {'key': 'value'}
        """
        self._push()

        if has_tag(obj, tags.REF):
            return self._pop(self._namedict.get(obj[tags.REF]))

        if has_tag(obj, tags.TYPE):
            typeref = loadclass(obj[tags.TYPE])
            if not typeref:
                return self._pop(obj)
            return self._pop(typeref)

        if has_tag(obj, tags.REPR):
            return self._pop(loadrepr(obj[tags.REPR]))

        if has_tag(obj, tags.OBJECT):

            cls = loadclass(obj[tags.OBJECT])
            if not cls:
                return self._pop(obj)
            try:
                instance = object.__new__(cls)
            except TypeError:
                # old-style classes
                try:
                    instance = cls()
                except TypeError:
                    # fail gracefully if the constructor requires arguments
                    self._mkref(obj)
                    return self._pop(obj)
            
            # keep a obj->name mapping for use in the _isobjref() case
            self._mkref(instance)

            for k, v in obj.iteritems():
                # ignore the reserved attribute
                if k in tags.RESERVED:
                    continue
                self._namestack.append(k)
                # step into the namespace
                value = self.restore(v)
                if (util.is_noncomplex(instance) or
                        util.is_dictionary_subclass(instance)):
                    instance[k] = value
                else:
                    instance.__dict__[k] = value
                # step out
                self._namestack.pop()
            return self._pop(instance)

        if util.is_list(obj):
            return self._pop([self.restore(v) for v in obj])

        if has_tag(obj, tags.TUPLE):
            return self._pop(tuple([self.restore(v) for v in obj[tags.TUPLE]]))

        if has_tag(obj, tags.SET):
            return self._pop(set([self.restore(v) for v in obj[tags.SET]]))

        if util.is_dictionary(obj):
            data = {}
            for k, v in obj.iteritems():
                self._namestack.append(k)
                data[k] = self.restore(v)
                self._namestack.pop()
            return self._pop(data)

        return self._pop(obj)