def default(obj): print("default called", obj) if isinstance(obj, array.array): typecode = 123 # application specific typecode data = tobytes(obj) return ExtType(typecode, data) raise TypeError("Unknown type object {obj!r}".format(obj=obj))
def default(obj): print('default called', obj) if isinstance(obj, array.array): typecode = 123 # application specific typecode data = tobytes(obj) return ExtType(typecode, data) raise TypeError("Unknown type object %r" % (obj, ))
def convert(values): """ convert the numpy values to a list """ dtype = values.dtype if is_categorical_dtype(values): return values elif is_object_dtype(dtype): return values.ravel().tolist() if needs_i8_conversion(dtype): values = values.view('i8') v = values.ravel() if compressor == 'zlib': zlib = import_optional_dependency( "zlib", extra="zlib is required when `compress='zlib'`." ) # return string arrays like they are if dtype == np.object_: return v.tolist() # convert to a bytes array v = v.tostring() return ExtType(0, zlib.compress(v)) elif compressor == 'blosc': blosc = import_optional_dependency( "blosc", extra="zlib is required when `compress='blosc'`." ) # return string arrays like they are if dtype == np.object_: return v.tolist() # convert to a bytes array v = v.tostring() return ExtType(0, blosc.compress(v, typesize=dtype.itemsize)) # ndarray (on original dtype) return ExtType(0, v.tostring())
def test_ext(): def check(ext, packed): assert packb(ext) == packed assert unpackb(packed) == ext check(ExtType(0x42, b"Z"), b"\xd4\x42Z") # fixext 1 check(ExtType(0x42, b"ZZ"), b"\xd5\x42ZZ") # fixext 2 check(ExtType(0x42, b"Z" * 4), b"\xd6\x42" + b"Z" * 4) # fixext 4 check(ExtType(0x42, b"Z" * 8), b"\xd7\x42" + b"Z" * 8) # fixext 8 check(ExtType(0x42, b"Z" * 16), b"\xd8\x42" + b"Z" * 16) # fixext 16 # ext 8 check(ExtType(0x42, b""), b"\xc7\x00\x42") check(ExtType(0x42, b"Z" * 255), b"\xc7\xff\x42" + b"Z" * 255) # ext 16 check(ExtType(0x42, b"Z" * 256), b"\xc8\x01\x00\x42" + b"Z" * 256) check(ExtType(0x42, b"Z" * 0xFFFF), b"\xc8\xff\xff\x42" + b"Z" * 0xFFFF) # ext 32 check(ExtType(0x42, b"Z" * 0x10000), b"\xc9\x00\x01\x00\x00\x42" + b"Z" * 0x10000)
def test_unpacker_ext_hook(self): class MyUnpacker(Unpacker): def __init__(self): super().__init__(ext_hook=self._hook, encoding="utf-8") def _hook(self, code, data): if code == 1: return int(data) else: return ExtType(code, data) unpacker = MyUnpacker() unpacker.feed(packb({"a": 1}, encoding="utf-8")) assert unpacker.unpack() == {"a": 1} unpacker.feed(packb({"a": ExtType(1, b"123")}, encoding="utf-8")) assert unpacker.unpack() == {"a": 123} unpacker.feed(packb({"a": ExtType(2, b"321")}, encoding="utf-8")) assert unpacker.unpack() == {"a": ExtType(2, b"321")}
def test_unpacker_ext_hook(self): class MyUnpacker(Unpacker): def __init__(self): super().__init__(ext_hook=self._hook, encoding='utf-8') def _hook(self, code, data): if code == 1: return int(data) else: return ExtType(code, data) unpacker = MyUnpacker() unpacker.feed(packb({'a': 1}, encoding='utf-8')) assert unpacker.unpack() == {'a': 1} unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8')) assert unpacker.unpack() == {'a': 123} unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8')) assert unpacker.unpack() == {'a': ExtType(2, b'321')}
def test_ext(): def check(ext, packed): assert packb(ext) == packed assert unpackb(packed) == ext check(ExtType(0x42, b'Z'), b'\xd4\x42Z') # fixext 1 check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ') # fixext 2 check(ExtType(0x42, b'Z' * 4), b'\xd6\x42' + b'Z' * 4) # fixext 4 check(ExtType(0x42, b'Z' * 8), b'\xd7\x42' + b'Z' * 8) # fixext 8 check(ExtType(0x42, b'Z' * 16), b'\xd8\x42' + b'Z' * 16) # fixext 16 # ext 8 check(ExtType(0x42, b''), b'\xc7\x00\x42') check(ExtType(0x42, b'Z' * 255), b'\xc7\xff\x42' + b'Z' * 255) # ext 16 check(ExtType(0x42, b'Z' * 256), b'\xc8\x01\x00\x42' + b'Z' * 256) check(ExtType(0x42, b'Z' * 0xffff), b'\xc8\xff\xff\x42' + b'Z' * 0xffff) # ext 32 check(ExtType(0x42, b'Z' * 0x10000), b'\xc9\x00\x01\x00\x00\x42' + b'Z' * 0x10000)
def test_max_ext_len(self): d = ExtType(42, b"abc") packed = packb(d) unpacker = Unpacker(max_ext_len=3) unpacker.feed(packed) assert unpacker.unpack() == d unpacker = Unpacker(max_ext_len=2) unpacker.feed(packed) pytest.raises(ValueError, unpacker.unpack)
def test_unpack_ext_type(): def check(b, expected): assert msgpack.unpackb(b) == expected check(b'\xd4\x42A', ExtType(0x42, b'A')) # fixext 1 check(b'\xd5\x42AB', ExtType(0x42, b'AB')) # fixext 2 check(b'\xd6\x42ABCD', ExtType(0x42, b'ABCD')) # fixext 4 check(b'\xd7\x42ABCDEFGH', ExtType(0x42, b'ABCDEFGH')) # fixext 8 check(b'\xd8\x42' + b'A' * 16, ExtType(0x42, b'A' * 16)) # fixext 16 check(b'\xc7\x03\x42ABC', ExtType(0x42, b'ABC')) # ext 8 check(b'\xc8\x01\x23\x42' + b'A' * 0x0123, ExtType(0x42, b'A' * 0x0123)) # ext 16 check(b'\xc9\x00\x01\x23\x45\x42' + b'A' * 0x00012345, ExtType(0x42, b'A' * 0x00012345)) # ext 32
def convert(values): """ convert the numpy values to a list """ dtype = values.dtype if is_categorical_dtype(values): return values elif is_object_dtype(dtype): return values.ravel().tolist() if needs_i8_conversion(dtype): values = values.view('i8') v = values.ravel() if compressor == 'zlib': _check_zlib() # return string arrays like they are if dtype == np.object_: return v.tolist() # convert to a bytes array v = v.tostring() return ExtType(0, zlib.compress(v)) elif compressor == 'blosc': _check_blosc() # return string arrays like they are if dtype == np.object_: return v.tolist() # convert to a bytes array v = v.tostring() return ExtType(0, blosc.compress(v, typesize=dtype.itemsize)) # ndarray (on original dtype) return ExtType(0, v.tostring())
def test_unpack_ext_type(): def check(b, expected): assert msgpack.unpackb(b) == expected check(b"\xd4\x42A", ExtType(0x42, b"A")) # fixext 1 check(b"\xd5\x42AB", ExtType(0x42, b"AB")) # fixext 2 check(b"\xd6\x42ABCD", ExtType(0x42, b"ABCD")) # fixext 4 check(b"\xd7\x42ABCDEFGH", ExtType(0x42, b"ABCDEFGH")) # fixext 8 check(b"\xd8\x42" + b"A" * 16, ExtType(0x42, b"A" * 16)) # fixext 16 check(b"\xc7\x03\x42ABC", ExtType(0x42, b"ABC")) # ext 8 check(b"\xc8\x01\x23\x42" + b"A" * 0x0123, ExtType(0x42, b"A" * 0x0123)) # ext 16 check( b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345, ExtType(0x42, b"A" * 0x00012345), ) # ext 32
def test_max_ext_len(self): d = ExtType(42, b"abc") packed = packb(d) unpacker = Unpacker(max_ext_len=3) unpacker.feed(packed) assert unpacker.unpack() == d unpacker = Unpacker(max_ext_len=2) unpacker.feed(packed) msg = "4 exceeds max_ext_len" with pytest.raises(ValueError, match=msg): unpacker.unpack()
def _hook(self, code, data): if code == 1: return int(data) else: return ExtType(code, data)