def default(obj):
if isinstance(obj, datetime.datetime):
return ExtType(EXT_CODE_DATETIME, pack_datetime(obj))
elif isinstance(obj, datetime.date):
return ExtType(EXT_CODE_DATE, pack_date(obj))
else:
raise TypeError("Unknown type: %r" % (obj, ))
def msgpack_ext_pack(x):
if isinstance(x, bd.Config):
return ExtType(1, pickle.dumps(x))
elif isinstance(x, Group):
return ExtType(2, pickle.dumps(str(x)))
elif isinstance(x, set):
return ExtType(3, pickle.dumps(list(x)))
return x
def _default(obj):
# TODO: we should be only testing against pendulum.Pendulum, but
# asyncpg returns datetime
if isinstance(obj, datetime):
return ExtType(1, struct_pack("!d", obj.timestamp()))
elif isinstance(obj, UUID):
return ExtType(2, obj.bytes)
raise TypeError("Unknown type: %r" % (obj, ))
def default(obj):
if isinstance(obj, (datetime, compat_datetime)):
return ExtType(
EXTID.datetime,
obj.isoformat().encode("utf-8")) # todo: use struct.Struct instead
elif isinstance(obj, set):
return ExtType(EXTID.set, packb(tuple(obj), use_bin_type=True))
elif isinstance(obj, frozenset):
return ExtType(EXTID.frozenset, packb(tuple(obj), use_bin_type=True))
return obj
def default(obj):
print('default called', obj)
if hasattr(obj, '_asdict'):
typecode = 123 # application specific typecode
data = packb([obj.__class__.__name__, obj._asdict().items()])
return ExtType(typecode, data)
raise TypeError("Unknwon type object %r" % (obj, ))
def default(o):
if isinstance(o, tuple):
# Convert to list and pack
payload = packb(
list(o), strict_types=True, use_bin_type=True, default=default)
return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload)
raise TypeError(repr(o))
def _unpackb_ext_hook(code, data):
if code == 1:
return Pendulum.utcfromtimestamp(struct_unpack("!d", data)[0])
elif code == 2:
return UUID(bytes=data)
return ExtType(code, data)
def default(obj):
print('default called', obj)
if isinstance(obj, array.array):
typecode = 123 # application specific typecode
data = obj.tostring()
return ExtType(typecode, data)
raise TypeError("Unknwon type object %r" % (obj,))
def _unpackb_ext_hook(code, data):
if code == 1:
# Uses UTC as default timezone
return from_timestamp(struct_unpack("!d", data)[0])
elif code == 2:
return UUID(bytes=data)
return ExtType(code, data)
def test_msgpack_exttype_passthrough(serializer):
serializer.register_custom_type(SlottedSimpleType)
ext = ExtType(6, b'somedata')
data = serializer.serialize(ext)
obj = serializer.deserialize(data)
assert isinstance(obj, ExtType)
assert obj.code == 6
assert obj.data == b'somedata'
def default(obj):
if type(obj) is uuid.UUID:
return ExtType(ExtTypes.UUID, obj.bytes)
if type(obj) is datetime:
return ExtType(ExtTypes.DATETIME, str(obj).encode('utf-8'))
if type(obj) is re._pattern_type:
return ExtType(ExtTypes.REGEX, obj.pattern.encode('utf-8'))
if type(obj) is set:
return list(obj)
if hasattr(obj, '__getstate__'):
return obj.__getstate__()
return str(obj)
def test_ext():
def check(ext, packed):
assert packb(ext) == packed
assert unpackb(packed) == ext
check(ExtType(0x42, b"Z"), b"\xd4\x42Z") # fixext 1
check(ExtType(0x42, b"ZZ"), b"\xd5\x42ZZ") # fixext 2
check(ExtType(0x42, b"Z" * 4), b"\xd6\x42" + b"Z" * 4) # fixext 4
check(ExtType(0x42, b"Z" * 8), b"\xd7\x42" + b"Z" * 8) # fixext 8
check(ExtType(0x42, b"Z" * 16), b"\xd8\x42" + b"Z" * 16) # fixext 16
# ext 8
check(ExtType(0x42, b""), b"\xc7\x00\x42")
check(ExtType(0x42, b"Z" * 255), b"\xc7\xff\x42" + b"Z" * 255)
# ext 16
check(ExtType(0x42, b"Z" * 256), b"\xc8\x01\x00\x42" + b"Z" * 256)
check(ExtType(0x42, b"Z" * 0xFFFF), b"\xc8\xff\xff\x42" + b"Z" * 0xFFFF)
# ext 32
check(ExtType(0x42, b"Z" * 0x10000), b"\xc9\x00\x01\x00\x00\x42" + b"Z" * 0x10000)
def test_unpacker_ext_hook():
class MyUnpacker(Unpacker):
def __init__(self):
super(MyUnpacker, self).__init__(ext_hook=self._hook, raw=False)
def _hook(self, code, data):
if code == 1:
return int(data)
else:
return ExtType(code, data)
unpacker = MyUnpacker()
unpacker.feed(packb({"a": 1}))
assert unpacker.unpack() == {"a": 1}
unpacker.feed(packb({"a": ExtType(1, b"123")}))
assert unpacker.unpack() == {"a": 123}
unpacker.feed(packb({"a": ExtType(2, b"321")}))
assert unpacker.unpack() == {"a": ExtType(2, b"321")}
def test_unpacker_ext_hook():
class MyUnpacker(Unpacker):
def __init__(self):
super(MyUnpacker, self).__init__(ext_hook=self._hook,
encoding='utf-8')
def _hook(self, code, data):
if code == 1:
return int(data)
else:
return ExtType(code, data)
unpacker = MyUnpacker()
unpacker.feed(packb({'a': 1}, encoding='utf-8'))
assert unpacker.unpack() == {'a': 1}
unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8'))
assert unpacker.unpack() == {'a': 123}
unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8'))
assert unpacker.unpack() == {'a': ExtType(2, b'321')}
def test_ext():
def check(ext, packed):
assert packb(ext) == packed
assert unpackb(packed) == ext
check(ExtType(0x42, b'Z'), b'\xd4\x42Z') # fixext 1
check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ') # fixext 2
check(ExtType(0x42, b'Z' * 4), b'\xd6\x42' + b'Z' * 4) # fixext 4
check(ExtType(0x42, b'Z' * 8), b'\xd7\x42' + b'Z' * 8) # fixext 8
check(ExtType(0x42, b'Z' * 16), b'\xd8\x42' + b'Z' * 16) # fixext 16
# ext 8
check(ExtType(0x42, b''), b'\xc7\x00\x42')
check(ExtType(0x42, b'Z' * 255), b'\xc7\xff\x42' + b'Z' * 255)
# ext 16
check(ExtType(0x42, b'Z' * 256), b'\xc8\x01\x00\x42' + b'Z' * 256)
check(ExtType(0x42, b'Z' * 0xffff), b'\xc8\xff\xff\x42' + b'Z' * 0xffff)
# ext 32
check(ExtType(0x42, b'Z' * 0x10000),
b'\xc9\x00\x01\x00\x00\x42' + b'Z' * 0x10000)
def default(obj):
print('default called', obj)
if isinstance(obj, array.array):
typecode = 123 # application specific typecode
try:
data = obj.tobytes()
except AttributeError:
data = obj.tostring()
return ExtType(typecode, data)
raise TypeError("Unknown type object %r" % (obj, ))
def test_preserve_resolution_order(self):
class A:
pass
class B(A):
pass
dump_a = mock.Mock(return_value=b"a")
load_a = mock.Mock(return_value=A())
dump_b = mock.Mock(return_value=b"b")
load_b = mock.Mock(return_value=B())
translation_table = {
1: (A, dump_a, load_a),
2: (B, dump_b, load_b),
}
packer = _Packer(translation_table=translation_table)
self.assertEqual(packer.packb(ExtType(1, b"a")), packer.packb(A()))
self.assertEqual(packer.packb(ExtType(2, b"b")), packer.packb(B()))
def _default_encoder(self, obj):
obj_type = obj.__class__
try:
typename, marshaller = self._marshallers[obj_type]
except KeyError:
raise LookupError('no marshaller found for type "{}"'.format(
obj_type.__class__.__name__)) from None
state = marshaller(obj)
data = typename + b':' + self.serialize(state)
return ExtType(self.custom_type_code, data)
def ext_type_pack_hook(self, obj, _sentinel=object()):
obj_class = obj.__class__
hit = self._pack_cache.get(obj_class, _sentinel)
if hit is None:
# packer has been not found by previous long-lookup
raise TypeError("Unknown type: {!r}".format(obj))
elif hit is _sentinel:
# do long-lookup
for code in sorted(self.translation_table):
cls, packer, unpacker = self.translation_table[code]
if isinstance(obj, cls):
self._pack_cache[obj_class] = (code, packer)
return ExtType(code, packer(obj))
else:
self._pack_cache[obj_class] = None
raise TypeError("Unknown type: {!r}".format(obj))
else:
# do shortcut
code, packer = hit
return ExtType(code, packer(obj))
def encode_nondefault_object(obj):
""" Encode an object by make it compatible with default msgpack encoder or using ExtType
:param obj: any objet
:return:
"""
if obj is None:
return
if hasattr(obj, '_asdict') and callable(obj._asdict):
return dict(obj._asdict())
elif hasattr(obj, 'to_dict') and callable(obj.to_dict):
return dict(obj.to_dict())
elif hasattr(obj, 'to_list') and callable(obj.to_list):
return list(obj.to_list())
elif isinstance(obj, dict): # handle Box, defaultdict and all variant of dictionary
return dict(obj)
elif isinstance(obj, (tuple, set, list)): # tuple,set,list will be treated as list
return list(obj)
elif isinstance(obj, Enum) and hasattr(obj, 'value'):
return obj.value
elif isinstance(obj, Constant) and hasattr(obj, '_value_'):
return obj._value_
elif isinstance(obj, Decimal):
return ExtType(ExternalType.DECIMAL, ensure_binary(str(obj)))
elif isinstance(obj, datetime):
return obj.strftime(DEFAULT_DATETIME_TIMEZONE_STRING_FORMAT)
elif isinstance(obj, date):
return obj.strftime(DEFAULT_DATE_STRING_FORMAT)
elif isinstance(obj, time):
return obj.strftime(DEFAULT_TIME_STRING_FORMAT)
elif isinstance(obj, timedelta):
if 0 <= obj.total_seconds() < 86400:
return '+{}'.format(obj)
return str(obj)
else:
if isinstance(obj, Model):
return ExtType(ExternalType.ORM_INSTANCE, pickle.dumps(obj, -1))
elif isinstance(obj, QuerySet):
return ExtType(ExternalType.ORM_QUERYSET, pickle.dumps((obj.model, obj.query), -1))
# logger.debug("unknown type obj=%s", obj)
return obj
def test_max_ext_len():
d = ExtType(42, b"abc")
packed = packb(d)
unpacker = Unpacker(max_ext_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_ext_len=2)
with pytest.raises(UnpackValueError):
unpacker.feed(packed)
unpacker.unpack()
def test_unpack_ext_type():
def check(b, expected):
assert msgpack.unpackb(b) == expected
check(b'\xd4\x42A', ExtType(0x42, b'A')) # fixext 1
check(b'\xd5\x42AB', ExtType(0x42, b'AB')) # fixext 2
check(b'\xd6\x42ABCD', ExtType(0x42, b'ABCD')) # fixext 4
check(b'\xd7\x42ABCDEFGH', ExtType(0x42, b'ABCDEFGH')) # fixext 8
check(b'\xd8\x42' + b'A'*16, ExtType(0x42, b'A'*16)) # fixext 16
check(b'\xc7\x03\x42ABC', ExtType(0x42, b'ABC')) # ext 8
check(b'\xc8\x01\x23\x42' + b'A'*0x0123,
ExtType(0x42, b'A'*0x0123)) # ext 16
check(b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345,
ExtType(0x42, b'A'*0x00012345)) # ext 32
def encode(obj):
if type(obj) in (list, tuple) or isinstance(obj, PVector):
return [encode(item) for item in obj]
if isinstance(obj, Mapping):
encoded_obj = {}
for key in obj.keys():
encoded_obj[encode(key)] = encode(obj[key])
return encoded_obj
if isinstance(obj, _native_builtin_types):
return obj
if isinstance(obj, Set):
return ExtType(
TYPE_PSET, packb([encode(item) for item in obj],
use_bin_type=True))
if isinstance(obj, PList):
return ExtType(
TYPE_PLIST, packb([encode(item) for item in obj],
use_bin_type=True))
if isinstance(obj, PBag):
return ExtType(
TYPE_PBAG, packb([encode(item) for item in obj],
use_bin_type=True))
if isinstance(obj, types.FunctionType):
return ExtType(
TYPE_FUNC, packb([obj.__module__, obj.__name__],
use_bin_type=True))
if isinstance(obj, Receiver):
return ExtType(TYPE_MBOX, packb(obj.encode(), use_bin_type=True))
# assume record
cls = obj.__class__
return ExtType(
0,
packb([cls.__module__, cls.__name__] + [encode(item) for item in obj],
use_bin_type=True))
def expand_grain_lazy(loader, node):
"""
Substitutes a variable read from a YAML node with a MsgPack ExtType value referring to data stored in a grain.
:param loader: YAML loader.
:type loader: yaml.loader.SafeLoader
:param node: Document node.
:type node: ScalarNode
:return: Corresponding value stored in the grain.
:rtype: msgpack.ExtType
"""
val = loader.construct_scalar(node)
return ExtType(_ext_types['grain'], val.encode('utf-8'))
def test_simple_translators(self):
translation_table = {
0: (Point, partial(dumps, protocol=HIGHEST_PROTOCOL), loads),
}
packer = _Packer(translation_table=translation_table)
pt = Point(1, 2)
data = dumps(pt, protocol=HIGHEST_PROTOCOL)
self.assertEqual(pt, packer.unpackb(packer.packb(pt)))
self.assertEqual(ExtType(0, data), packer.ext_type_pack_hook(pt))
self.assertEqual(pt, packer.ext_type_unpack_hook(0, data))
def test_override_translators(self):
translation_table = {
125: (Point, partial(dumps, protocol=HIGHEST_PROTOCOL), loads),
}
packer = _Packer(translation_table=translation_table)
pt = Point(3, 4)
data = dumps(pt, protocol=HIGHEST_PROTOCOL)
dt = datetime.time(15, 2)
self.assertEqual(ExtType(125, data), packer.ext_type_pack_hook(pt))
with self.assertRaisesRegex(TypeError, "Unknown type: "):
packer.ext_type_pack_hook(dt)
def test_unpack_ext_type():
def check(b, expected):
assert msgpack.unpackb(b) == expected
check(b"\xd4\x42A", ExtType(0x42, b"A")) # fixext 1
check(b"\xd5\x42AB", ExtType(0x42, b"AB")) # fixext 2
check(b"\xd6\x42ABCD", ExtType(0x42, b"ABCD")) # fixext 4
check(b"\xd7\x42ABCDEFGH", ExtType(0x42, b"ABCDEFGH")) # fixext 8
check(b"\xd8\x42" + b"A" * 16, ExtType(0x42, b"A" * 16)) # fixext 16
check(b"\xc7\x03\x42ABC", ExtType(0x42, b"ABC")) # ext 8
check(b"\xc8\x01\x23\x42" + b"A" * 0x0123,
ExtType(0x42, b"A" * 0x0123)) # ext 16
check(
b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345,
ExtType(0x42, b"A" * 0x00012345),
) # ext 32
def django_ext_hook(code, data):
if code == ExternalType.DECIMAL:
return Decimal(ensure_str(data, encoding='utf-8'), context=DEFAULT_DECIMAL_CONTEXT)
elif code == ExternalType.ORM_INSTANCE:
return pickle.loads(data)
elif code == ExternalType.ORM_QUERYSET:
# untouched queryset case
model, query = pickle.loads(data)
if isinstance(model, ModelBase) and isinstance(query, Query):
qs = model.objects.all()
qs.query = query
return qs
# unable to decode external type then return as it is
return ExtType(code, data)
def _custom_object_hook(self, code: int, data: bytes):
if code == self.custom_type_code:
typename, payload = data.split(b':', 1)
state = self.deserialize(payload)
try:
cls, unmarshaller = self._unmarshallers[typename]
except KeyError:
raise LookupError('no unmarshaller found for type "{}"'.format(
typename.decode('utf-8'))) from None
instance = cls.__new__(cls)
unmarshaller(instance, state)
return instance
else:
return ExtType(code, data)
def test_ext_type_errors(self):
packer = _Packer()
with self.assertRaisesRegex(TypeError, "Unknown type: "):
packer.ext_type_pack_hook(packer)
self.assertIn(_Packer, packer._pack_cache)
self.assertIsNone(packer._pack_cache[_Packer])
# lets try again just for good coverage
with self.assertRaisesRegex(TypeError, "Unknown type: "):
packer.ext_type_pack_hook(packer)
self.assertEqual(ExtType(1, b''), packer.ext_type_unpack_hook(1, b''))
# TODO: should be more specific errors
with self.assertRaises(Exception):
packer.ext_type_unpack_hook(127, b'bad data')