def default(obj): if isinstance(obj, datetime.datetime): return ExtType(EXT_CODE_DATETIME, pack_datetime(obj)) elif isinstance(obj, datetime.date): return ExtType(EXT_CODE_DATE, pack_date(obj)) else: raise TypeError("Unknown type: %r" % (obj, ))
def msgpack_ext_pack(x): if isinstance(x, bd.Config): return ExtType(1, pickle.dumps(x)) elif isinstance(x, Group): return ExtType(2, pickle.dumps(str(x))) elif isinstance(x, set): return ExtType(3, pickle.dumps(list(x))) return x
def _default(obj): # TODO: we should be only testing against pendulum.Pendulum, but # asyncpg returns datetime if isinstance(obj, datetime): return ExtType(1, struct_pack("!d", obj.timestamp())) elif isinstance(obj, UUID): return ExtType(2, obj.bytes) raise TypeError("Unknown type: %r" % (obj, ))
def default(obj): if isinstance(obj, (datetime, compat_datetime)): return ExtType( EXTID.datetime, obj.isoformat().encode("utf-8")) # todo: use struct.Struct instead elif isinstance(obj, set): return ExtType(EXTID.set, packb(tuple(obj), use_bin_type=True)) elif isinstance(obj, frozenset): return ExtType(EXTID.frozenset, packb(tuple(obj), use_bin_type=True)) return obj
def default(obj): print('default called', obj) if hasattr(obj, '_asdict'): typecode = 123 # application specific typecode data = packb([obj.__class__.__name__, obj._asdict().items()]) return ExtType(typecode, data) raise TypeError("Unknwon type object %r" % (obj, ))
def default(o): if isinstance(o, tuple): # Convert to list and pack payload = packb( list(o), strict_types=True, use_bin_type=True, default=default) return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload) raise TypeError(repr(o))
def _unpackb_ext_hook(code, data): if code == 1: return Pendulum.utcfromtimestamp(struct_unpack("!d", data)[0]) elif code == 2: return UUID(bytes=data) return ExtType(code, data)
def default(obj): print('default called', obj) if isinstance(obj, array.array): typecode = 123 # application specific typecode data = obj.tostring() return ExtType(typecode, data) raise TypeError("Unknwon type object %r" % (obj,))
def _unpackb_ext_hook(code, data): if code == 1: # Uses UTC as default timezone return from_timestamp(struct_unpack("!d", data)[0]) elif code == 2: return UUID(bytes=data) return ExtType(code, data)
def test_msgpack_exttype_passthrough(serializer): serializer.register_custom_type(SlottedSimpleType) ext = ExtType(6, b'somedata') data = serializer.serialize(ext) obj = serializer.deserialize(data) assert isinstance(obj, ExtType) assert obj.code == 6 assert obj.data == b'somedata'
def default(obj): if type(obj) is uuid.UUID: return ExtType(ExtTypes.UUID, obj.bytes) if type(obj) is datetime: return ExtType(ExtTypes.DATETIME, str(obj).encode('utf-8')) if type(obj) is re._pattern_type: return ExtType(ExtTypes.REGEX, obj.pattern.encode('utf-8')) if type(obj) is set: return list(obj) if hasattr(obj, '__getstate__'): return obj.__getstate__() return str(obj)
def test_ext(): def check(ext, packed): assert packb(ext) == packed assert unpackb(packed) == ext check(ExtType(0x42, b"Z"), b"\xd4\x42Z") # fixext 1 check(ExtType(0x42, b"ZZ"), b"\xd5\x42ZZ") # fixext 2 check(ExtType(0x42, b"Z" * 4), b"\xd6\x42" + b"Z" * 4) # fixext 4 check(ExtType(0x42, b"Z" * 8), b"\xd7\x42" + b"Z" * 8) # fixext 8 check(ExtType(0x42, b"Z" * 16), b"\xd8\x42" + b"Z" * 16) # fixext 16 # ext 8 check(ExtType(0x42, b""), b"\xc7\x00\x42") check(ExtType(0x42, b"Z" * 255), b"\xc7\xff\x42" + b"Z" * 255) # ext 16 check(ExtType(0x42, b"Z" * 256), b"\xc8\x01\x00\x42" + b"Z" * 256) check(ExtType(0x42, b"Z" * 0xFFFF), b"\xc8\xff\xff\x42" + b"Z" * 0xFFFF) # ext 32 check(ExtType(0x42, b"Z" * 0x10000), b"\xc9\x00\x01\x00\x00\x42" + b"Z" * 0x10000)
def test_unpacker_ext_hook(): class MyUnpacker(Unpacker): def __init__(self): super(MyUnpacker, self).__init__(ext_hook=self._hook, raw=False) def _hook(self, code, data): if code == 1: return int(data) else: return ExtType(code, data) unpacker = MyUnpacker() unpacker.feed(packb({"a": 1})) assert unpacker.unpack() == {"a": 1} unpacker.feed(packb({"a": ExtType(1, b"123")})) assert unpacker.unpack() == {"a": 123} unpacker.feed(packb({"a": ExtType(2, b"321")})) assert unpacker.unpack() == {"a": ExtType(2, b"321")}
def test_unpacker_ext_hook(): class MyUnpacker(Unpacker): def __init__(self): super(MyUnpacker, self).__init__(ext_hook=self._hook, encoding='utf-8') def _hook(self, code, data): if code == 1: return int(data) else: return ExtType(code, data) unpacker = MyUnpacker() unpacker.feed(packb({'a': 1}, encoding='utf-8')) assert unpacker.unpack() == {'a': 1} unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8')) assert unpacker.unpack() == {'a': 123} unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8')) assert unpacker.unpack() == {'a': ExtType(2, b'321')}
def test_ext(): def check(ext, packed): assert packb(ext) == packed assert unpackb(packed) == ext check(ExtType(0x42, b'Z'), b'\xd4\x42Z') # fixext 1 check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ') # fixext 2 check(ExtType(0x42, b'Z' * 4), b'\xd6\x42' + b'Z' * 4) # fixext 4 check(ExtType(0x42, b'Z' * 8), b'\xd7\x42' + b'Z' * 8) # fixext 8 check(ExtType(0x42, b'Z' * 16), b'\xd8\x42' + b'Z' * 16) # fixext 16 # ext 8 check(ExtType(0x42, b''), b'\xc7\x00\x42') check(ExtType(0x42, b'Z' * 255), b'\xc7\xff\x42' + b'Z' * 255) # ext 16 check(ExtType(0x42, b'Z' * 256), b'\xc8\x01\x00\x42' + b'Z' * 256) check(ExtType(0x42, b'Z' * 0xffff), b'\xc8\xff\xff\x42' + b'Z' * 0xffff) # ext 32 check(ExtType(0x42, b'Z' * 0x10000), b'\xc9\x00\x01\x00\x00\x42' + b'Z' * 0x10000)
def default(obj): print('default called', obj) if isinstance(obj, array.array): typecode = 123 # application specific typecode try: data = obj.tobytes() except AttributeError: data = obj.tostring() return ExtType(typecode, data) raise TypeError("Unknown type object %r" % (obj, ))
def test_preserve_resolution_order(self): class A: pass class B(A): pass dump_a = mock.Mock(return_value=b"a") load_a = mock.Mock(return_value=A()) dump_b = mock.Mock(return_value=b"b") load_b = mock.Mock(return_value=B()) translation_table = { 1: (A, dump_a, load_a), 2: (B, dump_b, load_b), } packer = _Packer(translation_table=translation_table) self.assertEqual(packer.packb(ExtType(1, b"a")), packer.packb(A())) self.assertEqual(packer.packb(ExtType(2, b"b")), packer.packb(B()))
def _default_encoder(self, obj): obj_type = obj.__class__ try: typename, marshaller = self._marshallers[obj_type] except KeyError: raise LookupError('no marshaller found for type "{}"'.format( obj_type.__class__.__name__)) from None state = marshaller(obj) data = typename + b':' + self.serialize(state) return ExtType(self.custom_type_code, data)
def ext_type_pack_hook(self, obj, _sentinel=object()): obj_class = obj.__class__ hit = self._pack_cache.get(obj_class, _sentinel) if hit is None: # packer has been not found by previous long-lookup raise TypeError("Unknown type: {!r}".format(obj)) elif hit is _sentinel: # do long-lookup for code in sorted(self.translation_table): cls, packer, unpacker = self.translation_table[code] if isinstance(obj, cls): self._pack_cache[obj_class] = (code, packer) return ExtType(code, packer(obj)) else: self._pack_cache[obj_class] = None raise TypeError("Unknown type: {!r}".format(obj)) else: # do shortcut code, packer = hit return ExtType(code, packer(obj))
def encode_nondefault_object(obj): """ Encode an object by make it compatible with default msgpack encoder or using ExtType :param obj: any objet :return: """ if obj is None: return if hasattr(obj, '_asdict') and callable(obj._asdict): return dict(obj._asdict()) elif hasattr(obj, 'to_dict') and callable(obj.to_dict): return dict(obj.to_dict()) elif hasattr(obj, 'to_list') and callable(obj.to_list): return list(obj.to_list()) elif isinstance(obj, dict): # handle Box, defaultdict and all variant of dictionary return dict(obj) elif isinstance(obj, (tuple, set, list)): # tuple,set,list will be treated as list return list(obj) elif isinstance(obj, Enum) and hasattr(obj, 'value'): return obj.value elif isinstance(obj, Constant) and hasattr(obj, '_value_'): return obj._value_ elif isinstance(obj, Decimal): return ExtType(ExternalType.DECIMAL, ensure_binary(str(obj))) elif isinstance(obj, datetime): return obj.strftime(DEFAULT_DATETIME_TIMEZONE_STRING_FORMAT) elif isinstance(obj, date): return obj.strftime(DEFAULT_DATE_STRING_FORMAT) elif isinstance(obj, time): return obj.strftime(DEFAULT_TIME_STRING_FORMAT) elif isinstance(obj, timedelta): if 0 <= obj.total_seconds() < 86400: return '+{}'.format(obj) return str(obj) else: if isinstance(obj, Model): return ExtType(ExternalType.ORM_INSTANCE, pickle.dumps(obj, -1)) elif isinstance(obj, QuerySet): return ExtType(ExternalType.ORM_QUERYSET, pickle.dumps((obj.model, obj.query), -1)) # logger.debug("unknown type obj=%s", obj) return obj
def test_max_ext_len(): d = ExtType(42, b"abc") packed = packb(d) unpacker = Unpacker(max_ext_len=3) unpacker.feed(packed) assert unpacker.unpack() == d unpacker = Unpacker(max_ext_len=2) with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack()
def test_unpack_ext_type(): def check(b, expected): assert msgpack.unpackb(b) == expected check(b'\xd4\x42A', ExtType(0x42, b'A')) # fixext 1 check(b'\xd5\x42AB', ExtType(0x42, b'AB')) # fixext 2 check(b'\xd6\x42ABCD', ExtType(0x42, b'ABCD')) # fixext 4 check(b'\xd7\x42ABCDEFGH', ExtType(0x42, b'ABCDEFGH')) # fixext 8 check(b'\xd8\x42' + b'A'*16, ExtType(0x42, b'A'*16)) # fixext 16 check(b'\xc7\x03\x42ABC', ExtType(0x42, b'ABC')) # ext 8 check(b'\xc8\x01\x23\x42' + b'A'*0x0123, ExtType(0x42, b'A'*0x0123)) # ext 16 check(b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345, ExtType(0x42, b'A'*0x00012345)) # ext 32
def encode(obj): if type(obj) in (list, tuple) or isinstance(obj, PVector): return [encode(item) for item in obj] if isinstance(obj, Mapping): encoded_obj = {} for key in obj.keys(): encoded_obj[encode(key)] = encode(obj[key]) return encoded_obj if isinstance(obj, _native_builtin_types): return obj if isinstance(obj, Set): return ExtType( TYPE_PSET, packb([encode(item) for item in obj], use_bin_type=True)) if isinstance(obj, PList): return ExtType( TYPE_PLIST, packb([encode(item) for item in obj], use_bin_type=True)) if isinstance(obj, PBag): return ExtType( TYPE_PBAG, packb([encode(item) for item in obj], use_bin_type=True)) if isinstance(obj, types.FunctionType): return ExtType( TYPE_FUNC, packb([obj.__module__, obj.__name__], use_bin_type=True)) if isinstance(obj, Receiver): return ExtType(TYPE_MBOX, packb(obj.encode(), use_bin_type=True)) # assume record cls = obj.__class__ return ExtType( 0, packb([cls.__module__, cls.__name__] + [encode(item) for item in obj], use_bin_type=True))
def expand_grain_lazy(loader, node): """ Substitutes a variable read from a YAML node with a MsgPack ExtType value referring to data stored in a grain. :param loader: YAML loader. :type loader: yaml.loader.SafeLoader :param node: Document node. :type node: ScalarNode :return: Corresponding value stored in the grain. :rtype: msgpack.ExtType """ val = loader.construct_scalar(node) return ExtType(_ext_types['grain'], val.encode('utf-8'))
def test_simple_translators(self): translation_table = { 0: (Point, partial(dumps, protocol=HIGHEST_PROTOCOL), loads), } packer = _Packer(translation_table=translation_table) pt = Point(1, 2) data = dumps(pt, protocol=HIGHEST_PROTOCOL) self.assertEqual(pt, packer.unpackb(packer.packb(pt))) self.assertEqual(ExtType(0, data), packer.ext_type_pack_hook(pt)) self.assertEqual(pt, packer.ext_type_unpack_hook(0, data))
def test_override_translators(self): translation_table = { 125: (Point, partial(dumps, protocol=HIGHEST_PROTOCOL), loads), } packer = _Packer(translation_table=translation_table) pt = Point(3, 4) data = dumps(pt, protocol=HIGHEST_PROTOCOL) dt = datetime.time(15, 2) self.assertEqual(ExtType(125, data), packer.ext_type_pack_hook(pt)) with self.assertRaisesRegex(TypeError, "Unknown type: "): packer.ext_type_pack_hook(dt)
def test_unpack_ext_type(): def check(b, expected): assert msgpack.unpackb(b) == expected check(b"\xd4\x42A", ExtType(0x42, b"A")) # fixext 1 check(b"\xd5\x42AB", ExtType(0x42, b"AB")) # fixext 2 check(b"\xd6\x42ABCD", ExtType(0x42, b"ABCD")) # fixext 4 check(b"\xd7\x42ABCDEFGH", ExtType(0x42, b"ABCDEFGH")) # fixext 8 check(b"\xd8\x42" + b"A" * 16, ExtType(0x42, b"A" * 16)) # fixext 16 check(b"\xc7\x03\x42ABC", ExtType(0x42, b"ABC")) # ext 8 check(b"\xc8\x01\x23\x42" + b"A" * 0x0123, ExtType(0x42, b"A" * 0x0123)) # ext 16 check( b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345, ExtType(0x42, b"A" * 0x00012345), ) # ext 32
def django_ext_hook(code, data): if code == ExternalType.DECIMAL: return Decimal(ensure_str(data, encoding='utf-8'), context=DEFAULT_DECIMAL_CONTEXT) elif code == ExternalType.ORM_INSTANCE: return pickle.loads(data) elif code == ExternalType.ORM_QUERYSET: # untouched queryset case model, query = pickle.loads(data) if isinstance(model, ModelBase) and isinstance(query, Query): qs = model.objects.all() qs.query = query return qs # unable to decode external type then return as it is return ExtType(code, data)
def _custom_object_hook(self, code: int, data: bytes): if code == self.custom_type_code: typename, payload = data.split(b':', 1) state = self.deserialize(payload) try: cls, unmarshaller = self._unmarshallers[typename] except KeyError: raise LookupError('no unmarshaller found for type "{}"'.format( typename.decode('utf-8'))) from None instance = cls.__new__(cls) unmarshaller(instance, state) return instance else: return ExtType(code, data)
def test_ext_type_errors(self): packer = _Packer() with self.assertRaisesRegex(TypeError, "Unknown type: "): packer.ext_type_pack_hook(packer) self.assertIn(_Packer, packer._pack_cache) self.assertIsNone(packer._pack_cache[_Packer]) # lets try again just for good coverage with self.assertRaisesRegex(TypeError, "Unknown type: "): packer.ext_type_pack_hook(packer) self.assertEqual(ExtType(1, b''), packer.ext_type_unpack_hook(1, b'')) # TODO: should be more specific errors with self.assertRaises(Exception): packer.ext_type_unpack_hook(127, b'bad data')