示例#1
0
 def test_integer(self):
     x = -(2 ** 63)
     assert unpackb(packb(x)) == x
     self.assertRaises((OverflowError, ValueError), packb, x - 1)
     x = 2 ** 64 - 1
     assert unpackb(packb(x)) == x
     self.assertRaises((OverflowError, ValueError), packb, x + 1)
示例#2
0
 def test_odict(self):
     seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
     od = OrderedDict(seq)
     assert unpackb(packb(od), use_list=1) == dict(seq)
     def pair_hook(seq):
         return list(seq)
     assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
示例#3
0
 def test_integer(self):
     x = -(2**63)
     assert unpackb(packb(x)) == x
     self.assertRaises((OverflowError, ValueError), packb, x - 1)
     x = 2**64 - 1
     assert unpackb(packb(x)) == x
     self.assertRaises((OverflowError, ValueError), packb, x + 1)
示例#4
0
    def test_odict(self):
        seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
        od = OrderedDict(seq)
        assert unpackb(packb(od), use_list=1) == dict(seq)

        def pair_hook(seq):
            return list(seq)

        assert unpackb(packb(od), object_pairs_hook=pair_hook,
                       use_list=1) == seq
示例#5
0
def test_str8():
    header = b'\xd9'
    data = b'x' * 32
    b = packb(data.decode(), use_bin_type=True)
    assert len(b) == len(data) + 2
    assert b[0:2] == header + b'\x20'
    assert b[2:] == data
    assert unpackb(b) == data

    data = b'x' * 255
    b = packb(data.decode(), use_bin_type=True)
    assert len(b) == len(data) + 2
    assert b[0:2] == header + b'\xff'
    assert b[2:] == data
    assert unpackb(b) == data
示例#6
0
def test_str8():
    header = b'\xd9'
    data = b'x' * 32
    b = packb(data.decode(), use_bin_type=True)
    assert len(b) == len(data) + 2
    assert b[0:2] == header + b'\x20'
    assert b[2:] == data
    assert unpackb(b) == data

    data = b'x' * 255
    b = packb(data.decode(), use_bin_type=True)
    assert len(b) == len(data) + 2
    assert b[0:2] == header + b'\xff'
    assert b[2:] == data
    assert unpackb(b) == data
示例#7
0
def test_bin8():
    header = b'\xc4'
    data = b''
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 2
    assert b[0:2] == header + b'\x00'
    assert b[2:] == data
    assert unpackb(b) == data

    data = b'x' * 255
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 2
    assert b[0:2] == header + b'\xff'
    assert b[2:] == data
    assert unpackb(b) == data
示例#8
0
def test_bin8():
    header = b'\xc4'
    data = b''
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 2
    assert b[0:2] == header + b'\x00'
    assert b[2:] == data
    assert unpackb(b) == data

    data = b'x' * 255
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 2
    assert b[0:2] == header + b'\xff'
    assert b[2:] == data
    assert unpackb(b) == data
示例#9
0
def deserialize(bytes, dtype, copy=False):
    if dtype == 'O':
        try:
            if msgpack.version >= (0, 5, 2):
                unpack_kwargs = {'raw': False}
            else:
                unpack_kwargs = {'encoding': 'utf-8'}

            blocks = [
                msgpack.unpackb(f, **unpack_kwargs) for f in framesplit(bytes)
            ]
        except Exception:
            blocks = [pickle.loads(f) for f in framesplit(bytes)]

        result = np.empty(sum(map(len, blocks)), dtype='O')
        i = 0
        for block in blocks:
            result[i:i + len(block)] = block
            i += len(block)
        return result
    else:
        result = np.frombuffer(bytes, dtype)
        if copy:
            result = result.copy()
        return result
示例#10
0
 def test_decode_pairs_hook(self):
     packed = packb([3, {1: 2, 3: 4}])
     prod_sum = 1 * 2 + 3 * 4
     unpacked = unpackb(
         packed, object_pairs_hook=lambda l: sum(k * v for k, v in l),
         use_list=1)
     assert unpacked[1] == prod_sum
示例#11
0
 def testIgnoreErrorsPack(self):
     re = unpackb(packb(compat.u("abcФФФdef"),
                        encoding='ascii',
                        unicode_errors='ignore'),
                  encoding='utf-8',
                  use_list=1)
     assert re == compat.u("abcdef")
示例#12
0
def test_bin16():
    header = b'\xc5'
    data = b'x' * 256
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 3
    assert b[0:1] == header
    assert b[1:3] == b'\x01\x00'
    assert b[3:] == data
    assert unpackb(b) == data

    data = b'x' * 65535
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 3
    assert b[0:1] == header
    assert b[1:3] == b'\xff\xff'
    assert b[3:] == data
    assert unpackb(b) == data
示例#13
0
def test_bin16():
    header = b'\xc5'
    data = b'x' * 256
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 3
    assert b[0:1] == header
    assert b[1:3] == b'\x01\x00'
    assert b[3:] == data
    assert unpackb(b) == data

    data = b'x' * 65535
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 3
    assert b[0:1] == header
    assert b[1:3] == b'\xff\xff'
    assert b[3:] == data
    assert unpackb(b) == data
示例#14
0
def test_bin32():
    header = b'\xc6'
    data = b'x' * 65536
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 5
    assert b[0:1] == header
    assert b[1:5] == b'\x00\x01\x00\x00'
    assert b[5:] == data
    assert unpackb(b) == data
示例#15
0
def loads(x):
    try:
        if msgpack.version >= (0, 5, 2):
            unpack_kwargs = {'raw': False}
        else:
            unpack_kwargs = {'encoding': 'utf-8'}
        return msgpack.unpackb(x, **unpack_kwargs)
    except:
        return pickle.loads(x)
示例#16
0
def test_bin32():
    header = b'\xc6'
    data = b'x' * 65536
    b = packb(data, use_bin_type=True)
    assert len(b) == len(data) + 5
    assert b[0:1] == header
    assert b[1:5] == b'\x00\x01\x00\x00'
    assert b[5:] == data
    assert unpackb(b) == data
示例#17
0
 def testPackUTF32(self):
     test_data = [
         compat.u(""),
         compat.u("abcd"),
         [compat.u("defgh")],
         compat.u("Русский текст"),
         ]
     for td in test_data:
         re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
         assert re == td
示例#18
0
 def testPackUTF32(self):
     test_data = [
         compat.u(""),
         compat.u("abcd"),
         [compat.u("defgh")],
         compat.u("Русский текст"),
     ]
     for td in test_data:
         re = unpackb(
             packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
         assert re == td
示例#19
0
 def testPackUnicode(self):
     test_data = [u(""), u("abcd"), [u("defgh")], u("Русский текст"), ]
     for td in test_data:
         re = unpackb(
             packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
         assert re == td
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
         re = Unpacker(
             compat.BytesIO(data), encoding='utf-8', use_list=1).unpack()
         assert re == td
示例#20
0
文件: core.py 项目: cowlicks/castra
def unpack_file(fn, encoding='utf8'):
    """ Unpack numpy array from filename

    Supports binary data with bloscpack and text data with msgpack+blosc

    >>> unpack_file('foo.blp')  # doctest: +SKIP
    array([1, 2, 3])

    See also:
        pack_file
    """
    try:
        return bloscpack.unpack_ndarray_file(fn)
    except ValueError:
        with open(fn, 'rb') as f:
            return np.array(msgpack.unpackb(blosc.decompress(f.read()),
                                            encoding=encoding))
示例#21
0
文件: core.py 项目: rockhowse/castra
def unpack_file(fn, encoding='utf8'):
    """ Unpack numpy array from filename

    Supports binary data with bloscpack and text data with msgpack+blosc

    >>> unpack_file('foo.blp')  # doctest: +SKIP
    array([1, 2, 3])

    See also:
        pack_file
    """
    try:
        return bloscpack.unpack_ndarray_file(fn)
    except ValueError:
        with open(fn, 'rb') as f:
            return np.array(
                msgpack.unpackb(blosc.decompress(f.read()), encoding=encoding))
示例#22
0
def test_extension_type():
    def default(obj):
        print('default called', obj)
        if isinstance(obj, array.array):
            typecode = 123  # application specific typecode
            data = obj.tostring()
            return ExtType(typecode, data)
        raise TypeError("Unknwon type object %r" % (obj, ))

    def ext_hook(code, data):
        print('ext_hook called', code, data)
        assert code == 123
        obj = array.array('d')
        obj.fromstring(data)
        return obj

    obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
    s = msgpack.packb(obj, default=default)
    obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
    assert obj == obj2
示例#23
0
def deserialize(bytes, dtype, copy=False):
    if dtype == 'O':
        try:
            blocks = [
                msgpack.unpackb(f, encoding='utf-8') for f in framesplit(bytes)
            ]
        except Exception:
            blocks = [pickle.loads(f) for f in framesplit(bytes)]

        result = np.empty(sum(map(len, blocks)), dtype='O')
        i = 0
        for block in blocks:
            result[i:i + len(block)] = block
            i += len(block)
        return result
    else:
        result = np.frombuffer(bytes, dtype)
        if copy:
            result = result.copy()
        return result
示例#24
0
def test_extension_type():
    def default(obj):
        print('default called', obj)
        if isinstance(obj, array.array):
            typecode = 123 # application specific typecode
            data = obj.tostring()
            return ExtType(typecode, data)
        raise TypeError("Unknwon type object %r" % (obj,))

    def ext_hook(code, data):
        print('ext_hook called', code, data)
        assert code == 123
        obj = array.array('d')
        obj.fromstring(data)
        return obj

    obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
    s = msgpack.packb(obj, default=default)
    obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
    assert obj == obj2
示例#25
0
文件: numpy.py 项目: dask/partd
def deserialize(bytes, dtype, copy=False):
    if dtype == 'O':
        try:
            if msgpack.version >= (0, 5, 2):
                unpack_kwargs = {'raw': False}
            else:
                unpack_kwargs = {'encoding': 'utf-8'}

            blocks = [msgpack.unpackb(f, **unpack_kwargs)
                      for f in framesplit(bytes)]
        except Exception:
            blocks = [pickle.loads(f) for f in framesplit(bytes)]

        result = np.empty(sum(map(len, blocks)), dtype='O')
        i = 0
        for block in blocks:
            result[i:i + len(block)] = block
            i += len(block)
        return result
    else:
        result = np.frombuffer(bytes, dtype)
        if copy:
            result = result.copy()
        return result
示例#26
0
def match(obj, buf):
    assert packb(obj) == buf
    assert unpackb(buf, use_list=0) == obj
示例#27
0
 def testIgnoreUnicodeErrors(self):
     re = unpackb(
         packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore',
         use_list=1)
     assert re == "abcdef"
示例#28
0
 def test_pairlist(self):
     pairlist = [(b'a', 1), (2, b'b'), (b'foo', b'bar')]
     packer = Packer()
     packed = packer.pack_map_pairs(pairlist)
     unpacked = unpackb(packed, object_pairs_hook=list)
     assert pairlist == unpacked
示例#29
0
def test_unicode():
    assert unpackb(packb('foobar'), use_list=1) == b'foobar'
示例#30
0
 def check(self, data, use_list=False):
     re = unpackb(packb(data), use_list=use_list)
     assert re == data
示例#31
0
 def testDecodeBinary(self):
     re = unpackb(packb("abc"), encoding=None, use_list=1)
     assert re == b"abc"
示例#32
0
 def test_array_hook(self):
     packed = packb([1, 2, 3])
     unpacked = unpackb(packed, list_hook=self._arr_to_str, use_list=1)
     assert unpacked == '123'
示例#33
0
 def testIgnoreErrorsPack(self):
     re = unpackb(
         packb(
             compat.u("abcФФФdef"), encoding='ascii',
             unicode_errors='ignore'), encoding='utf-8', use_list=1)
     assert re == compat.u("abcdef")
示例#34
0
 def check(ext, packed):
     assert packb(ext) == packed
     assert unpackb(packed) == ext
示例#35
0
 def f():
     packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}})
     unpackb(packed, object_hook=self.bad_complex_decoder)
示例#36
0
 def f():
     packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]})
     unpackb(packed, list_hook=self.bad_complex_decoder, use_list=1)
示例#37
0
def test_unpack_buffer():
    from array import array
    buf = array('b')
    buf.fromstring(packb(('foo', 'bar')))
    obj = unpackb(buf, use_list=1)
    assert [b'foo', b'bar'] == obj
示例#38
0
 def check(self, data, use_list=False):
     re = unpackb(packb(data), use_list=use_list)
     assert re == data
示例#39
0
 def test_decode_hook(self):
     packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
     unpacked = unpackb(packed,
                        object_hook=self._decode_complex,
                        use_list=1)
     assert unpacked[1] == 1 + 2j
示例#40
0
 def f():
     packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]})
     unpackb(packed, list_hook=self.bad_complex_decoder, use_list=1)
示例#41
0
 def f():
     packed = packb([3, 1 + 2j], default=lambda o: o)
     unpacked = unpackb(packed, use_list=1)
示例#42
0
 def testDecodeBinary(self):
     re = unpackb(packb("abc"), encoding=None, use_list=1)
     assert re == b"abc"
示例#43
0
 def f():
     packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}})
     unpackb(packed, object_hook=self.bad_complex_decoder)
示例#44
0
def loads(x):
    try:
        return msgpack.unpackb(x)
    except:
        return pickle.loads(x)
示例#45
0
def check(length, obj):
    v = packb(obj)
    assert len(v) == length, \
        "%r length should be %r but get %r" % (obj, length, len(v))
    assert unpackb(v, use_list=0) == obj
示例#46
0
def match(obj, buf):
    assert packb(obj) == buf
    assert unpackb(buf, use_list=0) == obj
示例#47
0
 def test_decode_hook(self):
     packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
     unpacked = unpackb(packed, object_hook=self._decode_complex,
                        use_list=1)
     assert unpacked[1] == 1 + 2j
示例#48
0
def test_unicode():
    assert unpackb(packb('foobar'), use_list=1) == b'foobar'
示例#49
0
 def test_encode_hook(self):
     packed = packb([3, 1 + 2j], default=self._encode_complex)
     unpacked = unpackb(packed, use_list=1)
     assert unpacked[1] == {b'__complex__': True, b'real': 1, b'imag': 2}
示例#50
0
 def test_array_hook(self):
     packed = packb([1, 2, 3])
     unpacked = unpackb(packed, list_hook=self._arr_to_str, use_list=1)
     assert unpacked == '123'
示例#51
0
 def testIgnoreUnicodeErrors(self):
     re = unpackb(packb(b'abc\xeddef'),
                  encoding='utf-8',
                  unicode_errors='ignore',
                  use_list=1)
     assert re == "abcdef"
示例#52
0
 def f():
     packed = packb([3, 1 + 2j], default=lambda o: o)
     unpacked = unpackb(packed, use_list=1)  # noqa
示例#53
0
def check(length, obj):
    v = packb(obj)
    assert len(v) == length, \
        "%r length should be %r but get %r" % (obj, length, len(v))
    assert unpackb(v, use_list=0) == obj
示例#54
0
 def test_encode_hook(self):
     packed = packb([3, 1 + 2j], default=self._encode_complex)
     unpacked = unpackb(packed, use_list=1)
     assert unpacked[1] == {b'__complex__': True, b'real': 1, b'imag': 2}
示例#55
0
 def check(b, expected):
     assert msgpack.unpackb(b) == expected
示例#56
0
 def test_pairlist(self):
     pairlist = [(b'a', 1), (2, b'b'), (b'foo', b'bar')]
     packer = Packer()
     packed = packer.pack_map_pairs(pairlist)
     unpacked = unpackb(packed, object_pairs_hook=list)
     assert pairlist == unpacked