def test_custom_extension_array():
import array
class ArrayExtension(bsdf.Extension):
name = 'array'
cls = array.array
def encode(self, s, arr):
return dict(typecode=str(arr.typecode), data=arr.tostring())
def decode(self, s, d):
a = array.array(d['typecode'])
a.fromstring(d['data'])
return a
extensions = [ArrayExtension]
a1 = [1, 2, array.array('b', [1, 2, 42])]
a2 = [1, 2, array.array('b', [1, 2, 42] * 1000)]
a3 = [1, 2, array.array('b', [4, 2, 42] * 1000)]
bb1 = bsdf.encode(a1, extensions)
bb2 = bsdf.encode(a2, extensions, compression=0)
bb3 = bsdf.encode(a3, extensions, compression=1)
assert len(bb2) > len(bb1) * 10
assert len(bb2) > len(bb3) * 10
b1 = bsdf.decode(bb1, extensions)
b2 = bsdf.decode(bb2, extensions)
b3 = bsdf.decode(bb3, extensions)
assert a1 == b1
assert a2 == b2
assert a3 == b3
def test_blob_writing1():
# Use blob to specify bytes
blob = bsdf.Blob(b'xxxx')
bb1 = bsdf.encode([2, 3, blob, 5])
# Again, with extra size
blob = bsdf.Blob(b'xxxx', extra_size=100)
bb2 = bsdf.encode([2, 3, blob, 5])
# Again, with checksum
blob = bsdf.Blob(b'xxxx', use_checksum=True)
bb3 = bsdf.encode([2, 3, blob, 5])
assert len(bb2) == len(bb1) + 100
assert len(bb3) == len(bb1) + 16
assert bsdf.decode(bb1) == [2, 3, b'xxxx', 5]
assert bsdf.decode(bb2) == [2, 3, b'xxxx', 5]
assert bsdf.decode(bb3) == [2, 3, b'xxxx', 5]
# Fail
with raises(TypeError):
bsdf.Blob([1, 2, 3])
with raises(RuntimeError):
blob.tell()
with raises(RuntimeError):
blob.seek(0)
with raises(RuntimeError):
blob.read(1)
with raises(RuntimeError):
blob.write(b'xx')
def test_standard_extensions_ndarray():
try:
import numpy as np
except ImportError:
skip('need numpy')
extensions = None
a1 = [1, 2, np.array([1, 2, 3, 4]).reshape((2, 2))]
a2 = [1, 2, np.array([1, 2, 42] * 1000)]
a3 = [1, 2, np.array([4, 2, 42] * 1000)]
b1 = bsdf.encode(a1, extensions)
b2 = bsdf.encode(a2, extensions, compression=0)
b3 = bsdf.encode(a3, extensions, compression=1)
assert len(b2) > len(b1) * 10
assert len(b2) > len(b3) * 10
c1 = bsdf.decode(b1, extensions)
c2 = bsdf.decode(b2, extensions)
c3 = bsdf.decode(b3, extensions)
assert np.all(a1[2] == c1[2])
assert np.all(a2[2] == c2[2])
assert np.all(a3[2] == c3[2])
def test_blob_reading1():
blob = bsdf.Blob(b'xxxx')
bb1 = bsdf.encode([2, 3, blob, 5])
res1 = bsdf.decode(bb1)
assert isinstance(res1[2], bytes)
res1 = bsdf.decode(bb1, lazy_blob=True)
assert not isinstance(res1[2], bytes) and isinstance(res1[2], bsdf.Blob)
res1[2].get_bytes() == b'xxxx'
def test_streaming_closing123():
""" Writing a streamed list, closing the stream. """
for iter in range(4):
f = io.BytesIO()
thelist = bsdf.ListStream()
bsdf.save(f, thelist)
thelist.append('hi')
# closing here will write the length of the stream, marking the stream as closed
if iter in (1, 2):
thelist.close()
for i in range(3):
thelist.append(i)
if iter == 2:
thelist.close()
if iter == 3:
thelist.close(True)
assert thelist.count == 4
assert thelist.index == thelist.count
bb = f.getvalue()
b1 = bsdf.decode(bb)
b2 = bsdf.decode(bb, load_streaming=True)
if iter in (0, 1, 2):
assert isinstance(b2, bsdf.ListStream) and not isinstance(b2, list)
else:
assert not isinstance(b2, bsdf.ListStream) and isinstance(b2, list)
if iter == 0:
# Not closed
assert b1 == ['hi', 0, 1, 2]
assert list(b2) == ['hi', 0, 1, 2]
elif iter == 1:
# Closed, and elements added later
assert b1 == ['hi']
assert list(b2) == ['hi']
elif iter == 2:
# Closed twice
assert b1 == ['hi', 0, 1, 2]
assert list(b2) == ['hi', 0, 1, 2]
elif iter == 3:
# Closed twice
assert b1 == ['hi', 0, 1, 2]
assert list(b2) == ['hi', 0, 1, 2]
else:
assert False
def test_liststreaming_reading1():
""" Reading a streamed list. """
f = io.BytesIO()
thelist = bsdf.ListStream()
a = [3, 4, thelist]
bsdf.save(f, a)
thelist.append('hi')
for i in range(3):
thelist.append(i)
bb = f.getvalue()
b = bsdf.decode(bb, load_streaming=True)
x = b[-1]
assert isinstance(x, bsdf.ListStream)
assert x.next() == 'hi'
assert x.next() == 0
assert x.next() == 1
assert x.next() == 2
with raises(StopIteration):
x.next()
# Support iteration
b = bsdf.decode(bb, load_streaming=True)
x = b[-1]
xx = [i for i in x]
assert xx == ['hi', 0, 1, 2]
# Cannot read after file is closed
f = io.BytesIO(bb)
b = bsdf.load(f, load_streaming=True)
x = b[-1]
f.close()
with raises(IOError):
x.next()
# Cannot read when in read-mode
ls = bsdf.ListStream()
with raises(IOError):
ls.next()
with raises(IOError):
list(ls)
# mmm, this would never happen, I guess, but need associated file!
ls = bsdf.ListStream('r')
with raises(IOError):
ls.next()
def test_float32():
# Using float32 makes smaller files
data = [2.0, 3.1, 5.1]
b1 = bsdf.encode(data, float64=False)
b2 = bsdf.encode(data, float64=True)
assert len(b1) < len(b2)
#
assert bsdf.decode(b1) != data
assert bsdf.decode(b2) == data
assert all(abs(i1 - i2) < 0.01 for i1, i2 in zip(bsdf.decode(b1), data))
assert all(abs(i1 - i2) < 0.001 for i1, i2 in zip(bsdf.decode(b2), data))
# Does not affect ints
data = [2, 3, 5]
b1 = bsdf.encode(data, float64=False)
b2 = bsdf.encode(data, float64=True)
assert len(b1) == len(b2)
#
assert bsdf.decode(b1) == data
assert bsdf.decode(b2) == data
# Ints are auto-scaled
b1 = bsdf.encode([3, 4, 5, 300, 400, 500])
b2 = bsdf.encode([300000, 400000, 500000, 3000000, 4000000, 5000000])
assert len(b1) < len(b2)
#
assert bsdf.decode(b1) == [3, 4, 5, 300, 400, 500]
assert bsdf.decode(b2) == [
300000, 400000, 500000, 3000000, 4000000, 5000000
]
def test_loaders_and_savers():
# load and save is already tested above and in many other places
s1 = dict(foo=42, bar=[1, 2.1, False, 'spam', b'eggs'])
# In-memory
bb = bsdf.encode(s1)
s2 = bsdf.decode(bb)
assert s1 == s2
# Using a filename
bsdf.save(tempfilename, s1)
s2 = bsdf.load(tempfilename)
assert s1 == s2
# Using a file object
with open(tempfilename, 'wb') as f:
bsdf.save(f, s1)
with open(tempfilename, 'rb') as f:
s2 = bsdf.load(f)
assert s1 == s2
# Using a very strict file object
with open(tempfilename, 'wb') as f:
bsdf.save(StrictWriteFile(f), s1)
with open(tempfilename, 'rb') as f:
s2 = bsdf.load(StrictReadFile(f))
assert s1 == s2
def test_all_types_simple():
s1 = dict(
v1=None,
v2=False,
v3=True,
v4=3,
v5=3.2,
v6=u'a', # u-prefix is needed in Legacy Python to not be bytes
v7=u'aa',
v8=(1, 2),
v9=[3, 4],
v10={
'a': 0,
'b': 1
},
v11=b'b',
v12=b'bb',
)
bb = bsdf.encode(s1)
s2 = bsdf.decode(bb)
# Correction - tuples become lists
assert isinstance(s2['v8'], list)
s2['v8'] = tuple(s2['v8'])
assert bb.startswith(b'BSDF')
assert s1 == s2
for key in s1:
assert type(s1[key]) == type(s2[key])
def load_bsdf(filename, decode=True):
''' load object from a BSDF file. automatically decode if in bytes. '''
object = bsdf.load(filename)
if decode and (type(object) is bytes):
return bsdf.decode(object)
else:
return object
def test_blob_modding2(): # with checksum
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2, use_checksum=True))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
blob.seek(4)
blob.write(b'aa')
blob.update_checksum()
assert bsdf.decode(f.getvalue()) == b'xxyyaa'
def test_extension_recurse():
class MyExt1(bsdf.Extension):
name = 'myob1'
cls = MyObject1
def encode(self, s, v):
return v.val
def decode(self, s, v):
return MyObject1(v)
class MyExt2(bsdf.Extension):
name = 'myob2'
cls = MyObject2
def encode(self, s, v):
# encode a MyObject2 as MyObject1
return MyObject1(v.val)
def decode(self, s, v):
# decode a MyObject2 from MyObject1
return MyObject2(v.val)
class MyExt3(bsdf.Extension):
name = 'myob2'
cls = MyObject2
def encode(self, s, v):
# encode a MyObject2 as [MyObject1]
return [MyObject1(v.val)]
def decode(self, s, v):
# decode a MyObject2 from MyObject1
return MyObject2(v[0].val)
a = MyObject2(14)
with raises(ValueError):
b = bsdf.encode(a, [MyExt1, MyExt2])
b = bsdf.encode(a, [MyExt1, MyExt3])
c = bsdf.decode(b, [MyExt1, MyExt3])
assert repr(a) == repr(c)
def test_liststreaming1():
""" Writing a streamed list. """
f = io.BytesIO()
thelist = bsdf.ListStream()
a = [3, 4, thelist]
bsdf.save(f, a)
thelist.append('hi')
for i in range(10):
thelist.append(i * 101)
thelist.append((4, 2))
thelist.close()
bb = f.getvalue()
b = bsdf.decode(bb)
assert b[-1] == [
'hi', 0, 101, 202, 303, 404, 505, 606, 707, 808, 909, [4, 2]
]
def test_extension_default_notimplemented():
class MyExt1(bsdf.Extension):
name = 'myob1'
cls = MyObject1
class MyExt2(bsdf.Extension):
name = 'myob1'
cls = MyObject1
def encode(self, s, v):
return v.val
assert 'myob1' in repr(MyExt1()) and 'BSDF extension' in repr(MyExt1())
a = MyObject1(7)
with raises(NotImplementedError):
bsdf.encode(a, [MyExt1])
b = bsdf.encode(a, [MyExt2])
with raises(NotImplementedError):
c = bsdf.decode(b, [MyExt2])
def test_parse_errors1():
msgs = []
class MyHandler(logging.Handler):
def emit(self, record):
msgs.append(record.getMessage())
myHandler = MyHandler()
logger = bsdf.logger.addHandler(myHandler)
V = bsdf.VERSION
assert V[0] > 0 and V[0] < 255 # or our tests will fail
assert V[1] > 0 and V[1] < 255
def header(*version):
return ('BSDF' + chr(version[0]) + chr(version[1])).encode()
assert bsdf.decode(header(*V) + b'v') == None
assert bsdf.decode(header(*V) + b'h\x07\x00') == 7
# Not BSDF
with raises(RuntimeError) as err:
assert bsdf.decode(b'BZDF\x02\x00v')
assert ' not ' in str(err) and 'BSDF' in str(err)
# Major version mismatch
with raises(RuntimeError) as err1:
assert bsdf.decode(header(V[0] - 1, V[1]) + b'v')
with raises(RuntimeError) as err2:
assert bsdf.decode(header(V[0] + 1, V[1]) + b'v')
for err in (err1, err2):
assert 'different major version' in str(err)
assert bsdf.__version__ in str(err)
# Smaller minor version is ok, larger minor version displays warning
out = ''
err = ''.join(msgs)
msgs[:] = []
bsdf.decode(header(V[0], V[1] - 1) + b'v')
out = ''
err = ''.join(msgs)
msgs[:] = []
assert not out and not err
bsdf.decode(header(V[0], V[1] + 1) + b'v')
out = ''
err = ''.join(msgs)
msgs[:] = []
assert not out and 'higher minor version' in err
# Wrong types
with raises(RuntimeError):
bsdf.decode(b'BSDF\x02\x00r\x07')
# \ r is not a known type
logger = bsdf.logger.removeFilter(myHandler)
def test_custom_extensions():
class MyExtension(bsdf.Extension):
name = 'myob'
def encode(self, s, v):
assert isinstance(s, bsdf.BsdfSerializer)
return v.val
def decode(self, s, v):
assert isinstance(s, bsdf.BsdfSerializer)
return MyObject1(v)
class MyExtension1(MyExtension):
cls = MyObject1
def match(self, s, v):
return False
class MyExtension2(MyExtension):
cls = MyObject1, MyObject2
def match(self, s, v):
return False
class MyExtension3(MyExtension):
cls = MyObject1
# default: def match(self, v): return isinstance(v, self.cls)
# Define data
a1 = [1, MyObject1(2), 3]
a2 = [1, MyObject1(2), MyObject2(3), 4]
# Extension 1 can only encode MyObject1
b1 = bsdf.encode(a1, [MyExtension1])
c1 = bsdf.decode(b1, [MyExtension1])
d1 = bsdf.decode(b1, [])
assert repr(a1) == repr(c1)
assert repr(a1) != repr(d1)
assert d1 == [1, 2, 3] # just has the undelying value
#
with raises(TypeError):
b2 = bsdf.encode(a2, [MyExtension1])
# Extension 2 can encode both
b1 = bsdf.encode(a1, [MyExtension2])
c1 = bsdf.decode(b1, [MyExtension2])
assert repr(a1) == repr(c1)
#
b2 = bsdf.encode(a2, [MyExtension2])
c2 = bsdf.decode(b2, [MyExtension2])
assert repr(a2).replace('ct2', 'ct1') == repr(c2)
# Extension 3 can encode both too
b3 = bsdf.encode(a1, [MyExtension2])
c3 = bsdf.decode(b1, [MyExtension2])
assert repr(a1) == repr(c1)
#
b3 = bsdf.encode(a2, [MyExtension3])
c3 = bsdf.decode(b2, [MyExtension3])
assert repr(a2).replace('ct2', 'ct1') == repr(c2)
# Overwriting works
b3 = bsdf.encode(a2, [MyExtension1, MyExtension3])
c3 = bsdf.decode(b2, [MyExtension1, MyExtension3])
assert repr(a2).replace('ct2', 'ct1') == repr(c2)
