def test_custom_extensions_fail():
with raises(TypeError):
bsdf.encode(None, ['not an extension'])
class MyExtension1(bsdf.Extension):
name = 3
with raises(TypeError):
bsdf.encode(None, [MyExtension1])
class MyExtension2(bsdf.Extension):
name = ''
with raises(NameError):
bsdf.encode(None, [MyExtension2])
class MyExtension3(bsdf.Extension):
name = 'x' * 1000
with raises(NameError):
bsdf.encode(None, [MyExtension3])
class MyExtension4(bsdf.Extension):
name = 'x'
cls = 4
with raises(TypeError):
bsdf.encode(None, [MyExtension4])
def test_custom_extension_array():
import array
class ArrayExtension(bsdf.Extension):
name = 'array'
cls = array.array
def encode(self, s, arr):
return dict(typecode=str(arr.typecode), data=arr.tostring())
def decode(self, s, d):
a = array.array(d['typecode'])
a.fromstring(d['data'])
return a
extensions = [ArrayExtension]
a1 = [1, 2, array.array('b', [1, 2, 42])]
a2 = [1, 2, array.array('b', [1, 2, 42] * 1000)]
a3 = [1, 2, array.array('b', [4, 2, 42] * 1000)]
bb1 = bsdf.encode(a1, extensions)
bb2 = bsdf.encode(a2, extensions, compression=0)
bb3 = bsdf.encode(a3, extensions, compression=1)
assert len(bb2) > len(bb1) * 10
assert len(bb2) > len(bb3) * 10
b1 = bsdf.decode(bb1, extensions)
b2 = bsdf.decode(bb2, extensions)
b3 = bsdf.decode(bb3, extensions)
assert a1 == b1
assert a2 == b2
assert a3 == b3
def test_blob_writing1():
# Use blob to specify bytes
blob = bsdf.Blob(b'xxxx')
bb1 = bsdf.encode([2, 3, blob, 5])
# Again, with extra size
blob = bsdf.Blob(b'xxxx', extra_size=100)
bb2 = bsdf.encode([2, 3, blob, 5])
# Again, with checksum
blob = bsdf.Blob(b'xxxx', use_checksum=True)
bb3 = bsdf.encode([2, 3, blob, 5])
assert len(bb2) == len(bb1) + 100
assert len(bb3) == len(bb1) + 16
assert bsdf.decode(bb1) == [2, 3, b'xxxx', 5]
assert bsdf.decode(bb2) == [2, 3, b'xxxx', 5]
assert bsdf.decode(bb3) == [2, 3, b'xxxx', 5]
# Fail
with raises(TypeError):
bsdf.Blob([1, 2, 3])
with raises(RuntimeError):
blob.tell()
with raises(RuntimeError):
blob.seek(0)
with raises(RuntimeError):
blob.read(1)
with raises(RuntimeError):
blob.write(b'xx')
def test_standard_extensions_ndarray():
try:
import numpy as np
except ImportError:
skip('need numpy')
extensions = None
a1 = [1, 2, np.array([1, 2, 3, 4]).reshape((2, 2))]
a2 = [1, 2, np.array([1, 2, 42] * 1000)]
a3 = [1, 2, np.array([4, 2, 42] * 1000)]
b1 = bsdf.encode(a1, extensions)
b2 = bsdf.encode(a2, extensions, compression=0)
b3 = bsdf.encode(a3, extensions, compression=1)
assert len(b2) > len(b1) * 10
assert len(b2) > len(b3) * 10
c1 = bsdf.decode(b1, extensions)
c2 = bsdf.decode(b2, extensions)
c3 = bsdf.decode(b3, extensions)
assert np.all(a1[2] == c1[2])
assert np.all(a2[2] == c2[2])
assert np.all(a3[2] == c3[2])
def test_detect_recursion1():
# Actually, we don't detect this, this will just raise a RecursionError
if 'recursion1' in os.getenv('BSDF_TEST_EXCLUDES', ''):
skip('recursion test explicitly skipped')
data = [3, 4]
data.append(data)
with raises(RuntimeError) as err:
bsdf.encode(data)
def xx_test_array_is_really_scalar():
skip('No, ppl should not encode such objects.')
# Will, work between Python session, but may not in other langs
try:
import numpy as np
except ImportError:
skip('need numpy')
a = np.array(12)
bb1 = bsdf.encode(a)
bb2 = bsdf.encode(12)
assert bb1 == bb2
def test_extension_recurse2():
class MyOb:
pass
class SillyExtension(bsdf.Extension):
name = 'silly'
cls = MyOb
def encode(self, s, v):
return v
with raises(ValueError) as err:
data = [MyOb()]
bsdf.encode(data, [SillyExtension])
def test_compression():
# Compressing makes smaller files
data = [1, 2, b'\x00' * 10000]
b1 = bsdf.encode(data, compression=0)
b2 = bsdf.encode(data, compression=1)
b3 = bsdf.encode(data, compression=2)
assert len(b1) > 10 * len(b2)
assert len(b1) > 10 * len(b3)
# Compression can be per-object, using blobs
data1 = [1, 2, b'\x00' * 10000]
data2 = [1, 2, bsdf.Blob(b'\x00' * 10000, compression=1)]
b1 = bsdf.encode(data1, compression=0)
b2 = bsdf.encode(data2, compression=0)
assert len(b1) > 10 * len(b2)
def test_loaders_and_savers():
# load and save is already tested above and in many other places
s1 = dict(foo=42, bar=[1, 2.1, False, 'spam', b'eggs'])
# In-memory
bb = bsdf.encode(s1)
s2 = bsdf.decode(bb)
assert s1 == s2
# Using a filename
bsdf.save(tempfilename, s1)
s2 = bsdf.load(tempfilename)
assert s1 == s2
# Using a file object
with open(tempfilename, 'wb') as f:
bsdf.save(f, s1)
with open(tempfilename, 'rb') as f:
s2 = bsdf.load(f)
assert s1 == s2
# Using a very strict file object
with open(tempfilename, 'wb') as f:
bsdf.save(StrictWriteFile(f), s1)
with open(tempfilename, 'rb') as f:
s2 = bsdf.load(StrictReadFile(f))
assert s1 == s2
def test_all_types_simple():
s1 = dict(
v1=None,
v2=False,
v3=True,
v4=3,
v5=3.2,
v6=u'a', # u-prefix is needed in Legacy Python to not be bytes
v7=u'aa',
v8=(1, 2),
v9=[3, 4],
v10={
'a': 0,
'b': 1
},
v11=b'b',
v12=b'bb',
)
bb = bsdf.encode(s1)
s2 = bsdf.decode(bb)
# Correction - tuples become lists
assert isinstance(s2['v8'], list)
s2['v8'] = tuple(s2['v8'])
assert bb.startswith(b'BSDF')
assert s1 == s2
for key in s1:
assert type(s1[key]) == type(s2[key])
def test_blob_reading2():
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
# Always seek first
blob.seek(0)
assert blob.read(2) == b'xx'
assert blob.tell() == 2
assert blob.read(2) == b'yy'
# We can overwrite, but changes an internal file that we cannot use :P
blob.write(b'aa')
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(-2) # relative to allocated size
assert blob.tell() == 6
# can just about do this, due to extra size
blob.seek(8)
# But this is too far
with raises(IOError):
blob.seek(9)
# And so is this
blob.seek(6)
with raises(IOError):
blob.write(b'xxx')
# And this
blob.seek(6)
with raises(IOError):
blob.read(3)
def test_blob_modding2(): # with checksum
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2, use_checksum=True))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
blob.seek(4)
blob.write(b'aa')
blob.update_checksum()
assert bsdf.decode(f.getvalue()) == b'xxyyaa'
def test_extension_recurse():
class MyExt1(bsdf.Extension):
name = 'myob1'
cls = MyObject1
def encode(self, s, v):
return v.val
def decode(self, s, v):
return MyObject1(v)
class MyExt2(bsdf.Extension):
name = 'myob2'
cls = MyObject2
def encode(self, s, v):
# encode a MyObject2 as MyObject1
return MyObject1(v.val)
def decode(self, s, v):
# decode a MyObject2 from MyObject1
return MyObject2(v.val)
class MyExt3(bsdf.Extension):
name = 'myob2'
cls = MyObject2
def encode(self, s, v):
# encode a MyObject2 as [MyObject1]
return [MyObject1(v.val)]
def decode(self, s, v):
# decode a MyObject2 from MyObject1
return MyObject2(v[0].val)
a = MyObject2(14)
with raises(ValueError):
b = bsdf.encode(a, [MyExt1, MyExt2])
b = bsdf.encode(a, [MyExt1, MyExt3])
c = bsdf.decode(b, [MyExt1, MyExt3])
assert repr(a) == repr(c)
def test_blob_reading3(): # compression
# ZLib
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=1))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
#
blob.get_bytes() == b'xxyyzz'
# BZ2
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=2))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
#
blob.get_bytes() == b'xxyyzz'
# But we cannot read or write
blob.seek(0)
with raises(IOError):
blob.read(2)
with raises(IOError):
blob.write(b'aa')
def test_blob_reading1():
blob = bsdf.Blob(b'xxxx')
bb1 = bsdf.encode([2, 3, blob, 5])
res1 = bsdf.decode(bb1)
assert isinstance(res1[2], bytes)
res1 = bsdf.decode(bb1, lazy_blob=True)
assert not isinstance(res1[2], bytes) and isinstance(res1[2], bsdf.Blob)
res1[2].get_bytes() == b'xxxx'
def test_float32():
# Using float32 makes smaller files
data = [2.0, 3.1, 5.1]
b1 = bsdf.encode(data, float64=False)
b2 = bsdf.encode(data, float64=True)
assert len(b1) < len(b2)
#
assert bsdf.decode(b1) != data
assert bsdf.decode(b2) == data
assert all(abs(i1 - i2) < 0.01 for i1, i2 in zip(bsdf.decode(b1), data))
assert all(abs(i1 - i2) < 0.001 for i1, i2 in zip(bsdf.decode(b2), data))
# Does not affect ints
data = [2, 3, 5]
b1 = bsdf.encode(data, float64=False)
b2 = bsdf.encode(data, float64=True)
assert len(b1) == len(b2)
#
assert bsdf.decode(b1) == data
assert bsdf.decode(b2) == data
# Ints are auto-scaled
b1 = bsdf.encode([3, 4, 5, 300, 400, 500])
b2 = bsdf.encode([300000, 400000, 500000, 3000000, 4000000, 5000000])
assert len(b1) < len(b2)
#
assert bsdf.decode(b1) == [3, 4, 5, 300, 400, 500]
assert bsdf.decode(b2) == [
300000, 400000, 500000, 3000000, 4000000, 5000000
]
def test_extension_default_notimplemented():
class MyExt1(bsdf.Extension):
name = 'myob1'
cls = MyObject1
class MyExt2(bsdf.Extension):
name = 'myob1'
cls = MyObject1
def encode(self, s, v):
return v.val
assert 'myob1' in repr(MyExt1()) and 'BSDF extension' in repr(MyExt1())
a = MyObject1(7)
with raises(NotImplementedError):
bsdf.encode(a, [MyExt1])
b = bsdf.encode(a, [MyExt2])
with raises(NotImplementedError):
c = bsdf.decode(b, [MyExt2])
def save_bsdf(filename, object, encode=True):
''' save object to a BSDF file. optionally BSDF-encoded to bytes. '''
if encode:
object = bsdf.encode(object) # type: bytes
bsdf.save(filename, object)
def test_view():
# Create file
data1 = [1, 2, 3, [4, 4, 4, 4, 4], 8, 9]
bsdf.save(tempfilename, data1)
# Test content, plain
r, e = run_local('view', tempfilename)
assert not e
assert tempfilename not in r
assert r.count('4') == 5
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, plus info
r, e = run_local('view', tempfilename, '--info')
assert not e
assert tempfilename in r
assert 'file_size:' in r
assert r.count('4') >= 5 # can also occur in meta info
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, max depth 1
r, e = run_local('view', tempfilename, '--depth=1')
assert not e
assert r.count('4') == 0 # collapsed
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, max depth 0
r, e = run_local('view', tempfilename, '--depth=0')
assert not e
assert r.count('\n') == 1 # all collapsed
assert '5' not in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Fail - not a file
r, e = run_local('view', '~/foo_does_not_exist.bsdf')
assert not r
assert 'invalid file' in e
# Fail - not a bsdf file
with open(tempfilename, 'wb') as f:
pass
r, e = run_local('view', tempfilename)
assert not r
assert 'does not look like a BSDF file' in e
# Fail - major version is off
bb = bsdf.encode(data1)
with open(tempfilename, 'wb') as f:
f.write(bb[:4] + b'\x00' + bb[5:])
r, e = run_local('view', tempfilename)
assert r == '' and 'different major version' in e
# Warn - minor version is lower than file
bb = bsdf.encode(data1)
with open(tempfilename, 'wb') as f:
f.write(bb[:5] + b'\xff' + bb[6:])
r, e = run_local('view', tempfilename)
assert not e
assert 'higher minor version' in r
assert r.count('4') >= 5
# Test string truncation
too_long = u'x' * 200
just_right = u'x' + '_' * 38 + 'x'
data1 = [too_long, just_right]
bsdf.save(tempfilename, data1)
r, e = run_local('view', tempfilename)
assert not e
assert too_long not in r and ('x' * 39 + u'\u2026') in r.replace(
'\\u2026', u'\u2026')
assert just_right in r
# Test float32 for cov
data1 = [3.14159, 42.0]
bsdf.save(tempfilename, data1, float64=False)
r, e = run_local('view', tempfilename)
assert not e
assert '3.14159' in r and '42.0' in r
# Test unclosed stream
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
with open(tempfilename, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
#
r, e = run_local('view', tempfilename)
assert not e
assert '6' in r and '7' in r
assert 'stream' in r and not '2' in r
def test_custom_extensions():
class MyExtension(bsdf.Extension):
name = 'myob'
def encode(self, s, v):
assert isinstance(s, bsdf.BsdfSerializer)
return v.val
def decode(self, s, v):
assert isinstance(s, bsdf.BsdfSerializer)
return MyObject1(v)
class MyExtension1(MyExtension):
cls = MyObject1
def match(self, s, v):
return False
class MyExtension2(MyExtension):
cls = MyObject1, MyObject2
def match(self, s, v):
return False
class MyExtension3(MyExtension):
cls = MyObject1
# default: def match(self, v): return isinstance(v, self.cls)
# Define data
a1 = [1, MyObject1(2), 3]
a2 = [1, MyObject1(2), MyObject2(3), 4]
# Extension 1 can only encode MyObject1
b1 = bsdf.encode(a1, [MyExtension1])
c1 = bsdf.decode(b1, [MyExtension1])
d1 = bsdf.decode(b1, [])
assert repr(a1) == repr(c1)
assert repr(a1) != repr(d1)
assert d1 == [1, 2, 3] # just has the undelying value
#
with raises(TypeError):
b2 = bsdf.encode(a2, [MyExtension1])
# Extension 2 can encode both
b1 = bsdf.encode(a1, [MyExtension2])
c1 = bsdf.decode(b1, [MyExtension2])
assert repr(a1) == repr(c1)
#
b2 = bsdf.encode(a2, [MyExtension2])
c2 = bsdf.decode(b2, [MyExtension2])
assert repr(a2).replace('ct2', 'ct1') == repr(c2)
# Extension 3 can encode both too
b3 = bsdf.encode(a1, [MyExtension2])
c3 = bsdf.decode(b1, [MyExtension2])
assert repr(a1) == repr(c1)
#
b3 = bsdf.encode(a2, [MyExtension3])
c3 = bsdf.decode(b2, [MyExtension3])
assert repr(a2).replace('ct2', 'ct1') == repr(c2)
# Overwriting works
b3 = bsdf.encode(a2, [MyExtension1, MyExtension3])
c3 = bsdf.decode(b2, [MyExtension1, MyExtension3])
assert repr(a2).replace('ct2', 'ct1') == repr(c2)
