def test_raise_from_object_hook(self):
def hook(obj):
raise DummyException
self.assertRaises(DummyException, unpackb, packb({}), object_hook=hook)
self.assertRaises(DummyException,
unpackb,
packb({'fizz': 'buzz'}),
object_hook=hook)
self.assertRaises(DummyException,
unpackb,
packb({'fizz': 'buzz'}),
object_pairs_hook=hook)
self.assertRaises(DummyException,
unpackb,
packb({'fizz': {
'buzz': 'spam'
}}),
object_hook=hook)
self.assertRaises(DummyException,
unpackb,
packb({'fizz': {
'buzz': 'spam'
}}),
object_pairs_hook=hook)
def test_odict(self):
seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
od = OrderedDict(seq)
assert unpackb(packb(od), use_list=1) == dict(seq)
def pair_hook(seq):
return list(seq)
assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
def test_integer(self):
x = -(2 ** 63)
assert unpackb(packb(x)) == x
self.assertRaises((OverflowError, ValueError), packb, x - 1)
x = 2 ** 64 - 1
assert unpackb(packb(x)) == x
self.assertRaises((OverflowError, ValueError), packb, x + 1)
def test_integer(self):
x = -(2**63)
assert unpackb(packb(x)) == x
self.assertRaises((OverflowError, ValueError), packb, x - 1)
x = 2**64 - 1
assert unpackb(packb(x)) == x
self.assertRaises((OverflowError, ValueError), packb, x + 1)
def test_raise_from_object_hook(self):
def hook(obj):
raise DummyException
self.assertRaises(DummyException, unpackb, packb({}), object_hook=hook)
self.assertRaises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_hook=hook)
self.assertRaises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_pairs_hook=hook)
self.assertRaises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_hook=hook)
self.assertRaises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_pairs_hook=hook)
def test_odict(self):
seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
od = OrderedDict(seq)
assert unpackb(packb(od), use_list=1) == dict(seq)
def pair_hook(seq):
return list(seq)
assert unpackb(packb(od), object_pairs_hook=pair_hook,
use_list=1) == seq
def test_str8():
header = b'\xd9'
data = b'x' * 32
b = packb(data.decode(), use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b'\x20'
assert b[2:] == data
assert unpackb(b) == data
data = b'x' * 255
b = packb(data.decode(), use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b'\xff'
assert b[2:] == data
assert unpackb(b) == data
def test_bin8():
header = b'\xc4'
data = b''
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b'\x00'
assert b[2:] == data
assert unpackb(b) == data
data = b'x' * 255
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b'\xff'
assert b[2:] == data
assert unpackb(b) == data
def test_bin8():
header = b'\xc4'
data = b''
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b'\x00'
assert b[2:] == data
assert unpackb(b) == data
data = b'x' * 255
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b'\xff'
assert b[2:] == data
assert unpackb(b) == data
def test_str8():
header = b'\xd9'
data = b'x' * 32
b = packb(data.decode(), use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b'\x20'
assert b[2:] == data
assert unpackb(b) == data
data = b'x' * 255
b = packb(data.decode(), use_bin_type=True)
assert len(b) == len(data) + 2
assert b[0:2] == header + b'\xff'
assert b[2:] == data
assert unpackb(b) == data
def test_decode_pairs_hook(self):
packed = packb([3, {1: 2, 3: 4}])
prod_sum = 1 * 2 + 3 * 4
unpacked = unpackb(
packed, object_pairs_hook=lambda l: sum(k * v for k, v in l),
use_list=1)
assert unpacked[1] == prod_sum
def serialize(x):
if x.dtype == 'O':
with ignoring(Exception): # Try msgpack (faster on strings)
return frame(msgpack.packb(x.tolist()))
return frame(pickle.dumps(x.tolist(), protocol=pickle.HIGHEST_PROTOCOL))
else:
return x.tobytes()
def testIgnoreErrorsPack(self):
re = unpackb(packb(compat.u("abcФФФdef"),
encoding='ascii',
unicode_errors='ignore'),
encoding='utf-8',
use_list=1)
assert re == compat.u("abcdef")
def test_incorrect_type_array():
unpacker = Unpacker()
unpacker.feed(packb(1))
try:
unpacker.read_array_header()
assert 0, 'should raise exception'
except UnexpectedTypeException:
assert 1, 'okay'
def test_correct_type_nested_array():
unpacker = Unpacker()
unpacker.feed(packb({'a': ['b', 'c', 'd']}))
try:
unpacker.read_array_header()
assert 0, 'should raise exception'
except UnexpectedTypeException:
assert 1, 'okay'
def serialize(x):
if x.dtype == 'O':
with ignoring(Exception): # Try msgpack (faster on strings)
return frame(msgpack.packb(x.tolist()))
return frame(pickle.dumps(x.tolist(),
protocol=pickle.HIGHEST_PROTOCOL))
else:
return x.tobytes()
def test_incorrect_type_nested_map():
unpacker = Unpacker()
unpacker.feed(packb([{'a': 'b'}]))
try:
unpacker.read_map_header()
assert 0, 'should raise exception'
except UnexpectedTypeException:
assert 1, 'okay'
def serialize(x):
if x.dtype == 'O':
l = x.flatten().tolist()
with ignoring(Exception): # Try msgpack (faster on strings)
return frame(msgpack.packb(l, use_bin_type=True))
return frame(pickle.dumps(l, protocol=pickle.HIGHEST_PROTOCOL))
else:
return x.tobytes()
def serialize(x):
if x.dtype == 'O':
l = x.flatten().tolist()
with ignoring(Exception): # Try msgpack (faster on strings)
return frame(msgpack.packb(l, use_bin_type=True))
return frame(pickle.dumps(l, protocol=pickle.HIGHEST_PROTOCOL))
else:
return x.tobytes()
def test_correct_type_nested_array():
unpacker = Unpacker()
unpacker.feed(packb({'a': ['b', 'c', 'd']}))
try:
unpacker.read_array_header()
assert 0, 'should raise exception'
except UnexpectedTypeException:
assert 1, 'okay'
def test_incorrect_type_nested_map():
unpacker = Unpacker()
unpacker.feed(packb([{'a': 'b'}]))
try:
unpacker.read_map_header()
assert 0, 'should raise exception'
except UnexpectedTypeException:
assert 1, 'okay'
def test_bin16():
header = b'\xc5'
data = b'x' * 256
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 3
assert b[0:1] == header
assert b[1:3] == b'\x01\x00'
assert b[3:] == data
assert unpackb(b) == data
data = b'x' * 65535
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 3
assert b[0:1] == header
assert b[1:3] == b'\xff\xff'
assert b[3:] == data
assert unpackb(b) == data
def test_bin16():
header = b'\xc5'
data = b'x' * 256
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 3
assert b[0:1] == header
assert b[1:3] == b'\x01\x00'
assert b[3:] == data
assert unpackb(b) == data
data = b'x' * 65535
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 3
assert b[0:1] == header
assert b[1:3] == b'\xff\xff'
assert b[3:] == data
assert unpackb(b) == data
def test_incorrect_type_array():
unpacker = Unpacker()
unpacker.feed(packb(1))
try:
unpacker.read_array_header()
assert 0, 'should raise exception'
except UnexpectedTypeException:
assert 1, 'okay'
def test_unpack_array_header_from_file(self):
f = BytesIO(packb([1, 2, 3, 4]))
unpacker = Unpacker(f)
assert unpacker.read_array_header() == 4
assert unpacker.unpack() == 1
assert unpacker.unpack() == 2
assert unpacker.unpack() == 3
assert unpacker.unpack() == 4
self.assertRaises(OutOfData, unpacker.unpack)
def test_unpack_array_header_from_file(self):
f = BytesIO(packb([1,2,3,4]))
unpacker = Unpacker(f)
assert unpacker.read_array_header() == 4
assert unpacker.unpack() == 1
assert unpacker.unpack() == 2
assert unpacker.unpack() == 3
assert unpacker.unpack() == 4
self.assertRaises(OutOfData, unpacker.unpack)
def test_bin32():
header = b'\xc6'
data = b'x' * 65536
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 5
assert b[0:1] == header
assert b[1:5] == b'\x00\x01\x00\x00'
assert b[5:] == data
assert unpackb(b) == data
def test_bin32():
header = b'\xc6'
data = b'x' * 65536
b = packb(data, use_bin_type=True)
assert len(b) == len(data) + 5
assert b[0:1] == header
assert b[1:5] == b'\x00\x01\x00\x00'
assert b[5:] == data
assert unpackb(b) == data
def test_write_bytes_multi_buffer():
long_val = (5) * 100
expected = packb(long_val)
unpacker = Unpacker(io.BytesIO(expected), read_size=3, max_buffer_size=3)
f = io.BytesIO()
unpacked = unpacker.unpack(f.write)
assert unpacked == long_val
assert f.getvalue() == expected
def test_unpacker_ext_hook(self):
class MyUnpacker(Unpacker):
def __init__(self):
super(MyUnpacker, self).__init__(ext_hook=self._hook,
encoding='utf-8')
def _hook(self, code, data):
if code == 1:
return int(data)
else:
return ExtType(code, data)
unpacker = MyUnpacker()
unpacker.feed(packb({'a': 1}, encoding='utf-8'))
assert unpacker.unpack() == {'a': 1}
unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8'))
assert unpacker.unpack() == {'a': 123}
unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8'))
assert unpacker.unpack() == {'a': ExtType(2, b'321')}
def testPackUTF32(self):
test_data = [
compat.u(""),
compat.u("abcd"),
[compat.u("defgh")],
compat.u("Русский текст"),
]
for td in test_data:
re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
assert re == td
def test_max_bin_len(self):
d = b'x' * 3
packed = packb(d, use_bin_type=True)
unpacker = Unpacker(max_bin_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_bin_len=2)
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def test_max_array_len(self):
d = [1, 2, 3]
packed = packb(d)
unpacker = Unpacker(max_array_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_array_len=2)
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def test_max_ext_len(self):
d = ExtType(42, b"abc")
packed = packb(d)
unpacker = Unpacker(max_ext_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_ext_len=2)
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def test_read_map_header():
unpacker = Unpacker()
unpacker.feed(packb({'a': 'A'}))
assert unpacker.read_map_header() == 1
assert unpacker.unpack() == B'a'
assert unpacker.unpack() == B'A'
try:
unpacker.unpack()
assert 0, 'should raise exception'
except OutOfData:
assert 1, 'okay'
def test_max_ext_len(self):
d = ExtType(42, b"abc")
packed = packb(d)
unpacker = Unpacker(max_ext_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_ext_len=2)
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def testPackUnicode(self):
test_data = [u(""), u("abcd"), [u("defgh")], u("Русский текст"), ]
for td in test_data:
re = unpackb(
packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
assert re == td
packer = Packer(encoding='utf-8')
data = packer.pack(td)
re = Unpacker(
compat.BytesIO(data), encoding='utf-8', use_list=1).unpack()
assert re == td
def test_unpacker_ext_hook(self):
class MyUnpacker(Unpacker):
def __init__(self):
super(MyUnpacker, self).__init__(ext_hook=self._hook,
encoding='utf-8')
def _hook(self, code, data):
if code == 1:
return int(data)
else:
return ExtType(code, data)
unpacker = MyUnpacker()
unpacker.feed(packb({'a': 1}, encoding='utf-8'))
assert unpacker.unpack() == {'a': 1}
unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8'))
assert unpacker.unpack() == {'a': 123}
unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8'))
assert unpacker.unpack() == {'a': ExtType(2, b'321')}
def test_max_str_len(self):
d = 'x' * 3
packed = packb(d)
unpacker = Unpacker(max_str_len=3, encoding='utf-8')
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_str_len=2, encoding='utf-8')
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def test_max_map_len(self):
d = {1: 2, 3: 4, 5: 6}
packed = packb(d)
unpacker = Unpacker(max_map_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_map_len=2)
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def test_max_map_len(self):
d = {1: 2, 3: 4, 5: 6}
packed = packb(d)
unpacker = Unpacker(max_map_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_map_len=2)
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def testPackUTF32(self):
test_data = [
compat.u(""),
compat.u("abcd"),
[compat.u("defgh")],
compat.u("Русский текст"),
]
for td in test_data:
re = unpackb(
packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
assert re == td
def test_max_array_len(self):
d = [1, 2, 3]
packed = packb(d)
unpacker = Unpacker(max_array_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_array_len=2)
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def test_max_bin_len(self):
d = b'x' * 3
packed = packb(d, use_bin_type=True)
unpacker = Unpacker(max_bin_len=3)
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_bin_len=2)
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def test_max_str_len(self):
d = 'x' * 3
packed = packb(d)
unpacker = Unpacker(max_str_len=3, encoding='utf-8')
unpacker.feed(packed)
assert unpacker.unpack() == d
unpacker = Unpacker(max_str_len=2, encoding='utf-8')
unpacker.feed(packed)
self.assertRaises(ValueError, unpacker.unpack)
def test_read_map_header():
unpacker = Unpacker()
unpacker.feed(packb({'a': 'A'}))
assert unpacker.read_map_header() == 1
assert unpacker.unpack() == B'a'
assert unpacker.unpack() == B'A'
try:
unpacker.unpack()
assert 0, 'should raise exception'
except OutOfData:
assert 1, 'okay'
def test_read_array_header():
unpacker = Unpacker()
unpacker.feed(packb(['a', 'b', 'c']))
assert unpacker.read_array_header() == 3
assert unpacker.unpack() == b'a'
assert unpacker.unpack() == b'b'
assert unpacker.unpack() == b'c'
try:
unpacker.unpack()
assert 0, 'should raise exception'
except OutOfData:
assert 1, 'okay'
def test_read_array_header():
unpacker = Unpacker()
unpacker.feed(packb(['a', 'b', 'c']))
assert unpacker.read_array_header() == 3
assert unpacker.unpack() == b'a'
assert unpacker.unpack() == b'b'
assert unpacker.unpack() == b'c'
try:
unpacker.unpack()
assert 0, 'should raise exception'
except OutOfData:
assert 1, 'okay'
def test_unpacker_hook_refcnt(self):
if not hasattr(sys, 'getrefcount'):
raise nose.SkipTest('no sys.getrefcount()')
result = []
def hook(x):
result.append(x)
return x
basecnt = sys.getrefcount(hook)
up = Unpacker(object_hook=hook, list_hook=hook)
assert sys.getrefcount(hook) >= basecnt + 2
up.feed(packb([{}]))
up.feed(packb([{}]))
assert up.unpack() == [{}]
assert up.unpack() == [{}]
assert result == [{}, [{}], {}, [{}]]
del up
assert sys.getrefcount(hook) == basecnt
def test_unpacker_hook_refcnt(self):
if not hasattr(sys, 'getrefcount'):
raise nose.SkipTest('no sys.getrefcount()')
result = []
def hook(x):
result.append(x)
return x
basecnt = sys.getrefcount(hook)
up = Unpacker(object_hook=hook, list_hook=hook)
assert sys.getrefcount(hook) >= basecnt + 2
up.feed(packb([{}]))
up.feed(packb([{}]))
assert up.unpack() == [{}]
assert up.unpack() == [{}]
assert result == [{}, [{}], {}, [{}]]
del up
assert sys.getrefcount(hook) == basecnt
def pack_file(x, fn, encoding='utf8'):
""" Pack numpy array into filename
Supports binary data with bloscpack and text data with msgpack+blosc
>>> pack_file(np.array([1, 2, 3]), 'foo.blp') # doctest: +SKIP
See also:
unpack_file
"""
if x.dtype != 'O':
bloscpack.pack_ndarray_file(x, fn)
else:
bytes = blosc.compress(msgpack.packb(x.tolist(), encoding=encoding), 1)
with open(fn, 'wb') as f:
f.write(bytes)
def pack_file(x, fn, encoding='utf8'):
""" Pack numpy array into filename
Supports binary data with bloscpack and text data with msgpack+blosc
>>> pack_file(np.array([1, 2, 3]), 'foo.blp') # doctest: +SKIP
See also:
unpack_file
"""
if x.dtype != 'O':
bloscpack.pack_ndarray_file(x, fn, bloscpack_args=bp_args,
blosc_args=blosc_args(x.dtype))
else:
bytes = blosc.compress(msgpack.packb(x.tolist(), encoding=encoding), 1)
with open(fn, 'wb') as f:
f.write(bytes)
def test_extension_type():
def default(obj):
print('default called', obj)
if isinstance(obj, array.array):
typecode = 123 # application specific typecode
data = obj.tostring()
return ExtType(typecode, data)
raise TypeError("Unknwon type object %r" % (obj,))
def ext_hook(code, data):
print('ext_hook called', code, data)
assert code == 123
obj = array.array('d')
obj.fromstring(data)
return obj
obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
s = msgpack.packb(obj, default=default)
obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
assert obj == obj2
def test_extension_type():
def default(obj):
print('default called', obj)
if isinstance(obj, array.array):
typecode = 123 # application specific typecode
data = obj.tostring()
return ExtType(typecode, data)
raise TypeError("Unknwon type object %r" % (obj, ))
def ext_hook(code, data):
print('ext_hook called', code, data)
assert code == 123
obj = array.array('d')
obj.fromstring(data)
return obj
obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
s = msgpack.packb(obj, default=default)
obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
assert obj == obj2
def dumps(x):
try:
return msgpack.packb(x, use_bin_type=True)
except:
return pickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
def test_unicode():
assert unpackb(packb('foobar'), use_list=1) == b'foobar'
def match(obj, buf):
assert packb(obj) == buf
assert unpackb(buf, use_list=0) == obj