def test_custom_extensions_fail(): with raises(TypeError): bsdf.encode(None, ['not an extension']) class MyExtension1(bsdf.Extension): name = 3 with raises(TypeError): bsdf.encode(None, [MyExtension1]) class MyExtension2(bsdf.Extension): name = '' with raises(NameError): bsdf.encode(None, [MyExtension2]) class MyExtension3(bsdf.Extension): name = 'x' * 1000 with raises(NameError): bsdf.encode(None, [MyExtension3]) class MyExtension4(bsdf.Extension): name = 'x' cls = 4 with raises(TypeError): bsdf.encode(None, [MyExtension4])
def test_custom_extension_array(): import array class ArrayExtension(bsdf.Extension): name = 'array' cls = array.array def encode(self, s, arr): return dict(typecode=str(arr.typecode), data=arr.tostring()) def decode(self, s, d): a = array.array(d['typecode']) a.fromstring(d['data']) return a extensions = [ArrayExtension] a1 = [1, 2, array.array('b', [1, 2, 42])] a2 = [1, 2, array.array('b', [1, 2, 42] * 1000)] a3 = [1, 2, array.array('b', [4, 2, 42] * 1000)] bb1 = bsdf.encode(a1, extensions) bb2 = bsdf.encode(a2, extensions, compression=0) bb3 = bsdf.encode(a3, extensions, compression=1) assert len(bb2) > len(bb1) * 10 assert len(bb2) > len(bb3) * 10 b1 = bsdf.decode(bb1, extensions) b2 = bsdf.decode(bb2, extensions) b3 = bsdf.decode(bb3, extensions) assert a1 == b1 assert a2 == b2 assert a3 == b3
def test_blob_writing1(): # Use blob to specify bytes blob = bsdf.Blob(b'xxxx') bb1 = bsdf.encode([2, 3, blob, 5]) # Again, with extra size blob = bsdf.Blob(b'xxxx', extra_size=100) bb2 = bsdf.encode([2, 3, blob, 5]) # Again, with checksum blob = bsdf.Blob(b'xxxx', use_checksum=True) bb3 = bsdf.encode([2, 3, blob, 5]) assert len(bb2) == len(bb1) + 100 assert len(bb3) == len(bb1) + 16 assert bsdf.decode(bb1) == [2, 3, b'xxxx', 5] assert bsdf.decode(bb2) == [2, 3, b'xxxx', 5] assert bsdf.decode(bb3) == [2, 3, b'xxxx', 5] # Fail with raises(TypeError): bsdf.Blob([1, 2, 3]) with raises(RuntimeError): blob.tell() with raises(RuntimeError): blob.seek(0) with raises(RuntimeError): blob.read(1) with raises(RuntimeError): blob.write(b'xx')
def test_standard_extensions_ndarray(): try: import numpy as np except ImportError: skip('need numpy') extensions = None a1 = [1, 2, np.array([1, 2, 3, 4]).reshape((2, 2))] a2 = [1, 2, np.array([1, 2, 42] * 1000)] a3 = [1, 2, np.array([4, 2, 42] * 1000)] b1 = bsdf.encode(a1, extensions) b2 = bsdf.encode(a2, extensions, compression=0) b3 = bsdf.encode(a3, extensions, compression=1) assert len(b2) > len(b1) * 10 assert len(b2) > len(b3) * 10 c1 = bsdf.decode(b1, extensions) c2 = bsdf.decode(b2, extensions) c3 = bsdf.decode(b3, extensions) assert np.all(a1[2] == c1[2]) assert np.all(a2[2] == c2[2]) assert np.all(a3[2] == c3[2])
def test_detect_recursion1(): # Actually, we don't detect this, this will just raise a RecursionError if 'recursion1' in os.getenv('BSDF_TEST_EXCLUDES', ''): skip('recursion test explicitly skipped') data = [3, 4] data.append(data) with raises(RuntimeError) as err: bsdf.encode(data)
def xx_test_array_is_really_scalar(): skip('No, ppl should not encode such objects.') # Will, work between Python session, but may not in other langs try: import numpy as np except ImportError: skip('need numpy') a = np.array(12) bb1 = bsdf.encode(a) bb2 = bsdf.encode(12) assert bb1 == bb2
def test_extension_recurse2(): class MyOb: pass class SillyExtension(bsdf.Extension): name = 'silly' cls = MyOb def encode(self, s, v): return v with raises(ValueError) as err: data = [MyOb()] bsdf.encode(data, [SillyExtension])
def test_compression(): # Compressing makes smaller files data = [1, 2, b'\x00' * 10000] b1 = bsdf.encode(data, compression=0) b2 = bsdf.encode(data, compression=1) b3 = bsdf.encode(data, compression=2) assert len(b1) > 10 * len(b2) assert len(b1) > 10 * len(b3) # Compression can be per-object, using blobs data1 = [1, 2, b'\x00' * 10000] data2 = [1, 2, bsdf.Blob(b'\x00' * 10000, compression=1)] b1 = bsdf.encode(data1, compression=0) b2 = bsdf.encode(data2, compression=0) assert len(b1) > 10 * len(b2)
def test_loaders_and_savers(): # load and save is already tested above and in many other places s1 = dict(foo=42, bar=[1, 2.1, False, 'spam', b'eggs']) # In-memory bb = bsdf.encode(s1) s2 = bsdf.decode(bb) assert s1 == s2 # Using a filename bsdf.save(tempfilename, s1) s2 = bsdf.load(tempfilename) assert s1 == s2 # Using a file object with open(tempfilename, 'wb') as f: bsdf.save(f, s1) with open(tempfilename, 'rb') as f: s2 = bsdf.load(f) assert s1 == s2 # Using a very strict file object with open(tempfilename, 'wb') as f: bsdf.save(StrictWriteFile(f), s1) with open(tempfilename, 'rb') as f: s2 = bsdf.load(StrictReadFile(f)) assert s1 == s2
def test_all_types_simple(): s1 = dict( v1=None, v2=False, v3=True, v4=3, v5=3.2, v6=u'a', # u-prefix is needed in Legacy Python to not be bytes v7=u'aa', v8=(1, 2), v9=[3, 4], v10={ 'a': 0, 'b': 1 }, v11=b'b', v12=b'bb', ) bb = bsdf.encode(s1) s2 = bsdf.decode(bb) # Correction - tuples become lists assert isinstance(s2['v8'], list) s2['v8'] = tuple(s2['v8']) assert bb.startswith(b'BSDF') assert s1 == s2 for key in s1: assert type(s1[key]) == type(s2[key])
def test_blob_reading2(): bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2)) f = io.BytesIO(bb) blob = bsdf.load(f, lazy_blob=True) # Always seek first blob.seek(0) assert blob.read(2) == b'xx' assert blob.tell() == 2 assert blob.read(2) == b'yy' # We can overwrite, but changes an internal file that we cannot use :P blob.write(b'aa') blob.seek(0) assert blob.read(2) == b'xx' blob.seek(-2) # relative to allocated size assert blob.tell() == 6 # can just about do this, due to extra size blob.seek(8) # But this is too far with raises(IOError): blob.seek(9) # And so is this blob.seek(6) with raises(IOError): blob.write(b'xxx') # And this blob.seek(6) with raises(IOError): blob.read(3)
def test_blob_modding2(): # with checksum bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2, use_checksum=True)) f = io.BytesIO(bb) blob = bsdf.load(f, lazy_blob=True) blob.seek(4) blob.write(b'aa') blob.update_checksum() assert bsdf.decode(f.getvalue()) == b'xxyyaa'
def test_extension_recurse(): class MyExt1(bsdf.Extension): name = 'myob1' cls = MyObject1 def encode(self, s, v): return v.val def decode(self, s, v): return MyObject1(v) class MyExt2(bsdf.Extension): name = 'myob2' cls = MyObject2 def encode(self, s, v): # encode a MyObject2 as MyObject1 return MyObject1(v.val) def decode(self, s, v): # decode a MyObject2 from MyObject1 return MyObject2(v.val) class MyExt3(bsdf.Extension): name = 'myob2' cls = MyObject2 def encode(self, s, v): # encode a MyObject2 as [MyObject1] return [MyObject1(v.val)] def decode(self, s, v): # decode a MyObject2 from MyObject1 return MyObject2(v[0].val) a = MyObject2(14) with raises(ValueError): b = bsdf.encode(a, [MyExt1, MyExt2]) b = bsdf.encode(a, [MyExt1, MyExt3]) c = bsdf.decode(b, [MyExt1, MyExt3]) assert repr(a) == repr(c)
def test_blob_reading3(): # compression # ZLib bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=1)) f = io.BytesIO(bb) blob = bsdf.load(f, lazy_blob=True) # blob.get_bytes() == b'xxyyzz' # BZ2 bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=2)) f = io.BytesIO(bb) blob = bsdf.load(f, lazy_blob=True) # blob.get_bytes() == b'xxyyzz' # But we cannot read or write blob.seek(0) with raises(IOError): blob.read(2) with raises(IOError): blob.write(b'aa')
def test_blob_reading1(): blob = bsdf.Blob(b'xxxx') bb1 = bsdf.encode([2, 3, blob, 5]) res1 = bsdf.decode(bb1) assert isinstance(res1[2], bytes) res1 = bsdf.decode(bb1, lazy_blob=True) assert not isinstance(res1[2], bytes) and isinstance(res1[2], bsdf.Blob) res1[2].get_bytes() == b'xxxx'
def test_float32(): # Using float32 makes smaller files data = [2.0, 3.1, 5.1] b1 = bsdf.encode(data, float64=False) b2 = bsdf.encode(data, float64=True) assert len(b1) < len(b2) # assert bsdf.decode(b1) != data assert bsdf.decode(b2) == data assert all(abs(i1 - i2) < 0.01 for i1, i2 in zip(bsdf.decode(b1), data)) assert all(abs(i1 - i2) < 0.001 for i1, i2 in zip(bsdf.decode(b2), data)) # Does not affect ints data = [2, 3, 5] b1 = bsdf.encode(data, float64=False) b2 = bsdf.encode(data, float64=True) assert len(b1) == len(b2) # assert bsdf.decode(b1) == data assert bsdf.decode(b2) == data # Ints are auto-scaled b1 = bsdf.encode([3, 4, 5, 300, 400, 500]) b2 = bsdf.encode([300000, 400000, 500000, 3000000, 4000000, 5000000]) assert len(b1) < len(b2) # assert bsdf.decode(b1) == [3, 4, 5, 300, 400, 500] assert bsdf.decode(b2) == [ 300000, 400000, 500000, 3000000, 4000000, 5000000 ]
def test_extension_default_notimplemented(): class MyExt1(bsdf.Extension): name = 'myob1' cls = MyObject1 class MyExt2(bsdf.Extension): name = 'myob1' cls = MyObject1 def encode(self, s, v): return v.val assert 'myob1' in repr(MyExt1()) and 'BSDF extension' in repr(MyExt1()) a = MyObject1(7) with raises(NotImplementedError): bsdf.encode(a, [MyExt1]) b = bsdf.encode(a, [MyExt2]) with raises(NotImplementedError): c = bsdf.decode(b, [MyExt2])
def save_bsdf(filename, object, encode=True): ''' save object to a BSDF file. optionally BSDF-encoded to bytes. ''' if encode: object = bsdf.encode(object) # type: bytes bsdf.save(filename, object)
def test_view(): # Create file data1 = [1, 2, 3, [4, 4, 4, 4, 4], 8, 9] bsdf.save(tempfilename, data1) # Test content, plain r, e = run_local('view', tempfilename) assert not e assert tempfilename not in r assert r.count('4') == 5 assert '5' in r # number of elements in inner list assert '6' in r # number of elements in outer list # Test content, plus info r, e = run_local('view', tempfilename, '--info') assert not e assert tempfilename in r assert 'file_size:' in r assert r.count('4') >= 5 # can also occur in meta info assert '5' in r # number of elements in inner list assert '6' in r # number of elements in outer list # Test content, max depth 1 r, e = run_local('view', tempfilename, '--depth=1') assert not e assert r.count('4') == 0 # collapsed assert '5' in r # number of elements in inner list assert '6' in r # number of elements in outer list # Test content, max depth 0 r, e = run_local('view', tempfilename, '--depth=0') assert not e assert r.count('\n') == 1 # all collapsed assert '5' not in r # number of elements in inner list assert '6' in r # number of elements in outer list # Fail - not a file r, e = run_local('view', '~/foo_does_not_exist.bsdf') assert not r assert 'invalid file' in e # Fail - not a bsdf file with open(tempfilename, 'wb') as f: pass r, e = run_local('view', tempfilename) assert not r assert 'does not look like a BSDF file' in e # Fail - major version is off bb = bsdf.encode(data1) with open(tempfilename, 'wb') as f: f.write(bb[:4] + b'\x00' + bb[5:]) r, e = run_local('view', tempfilename) assert r == '' and 'different major version' in e # Warn - minor version is lower than file bb = bsdf.encode(data1) with open(tempfilename, 'wb') as f: f.write(bb[:5] + b'\xff' + bb[6:]) r, e = run_local('view', tempfilename) assert not e assert 'higher minor version' in r assert r.count('4') >= 5 # Test string truncation too_long = u'x' * 200 just_right = u'x' + '_' * 38 + 'x' data1 = [too_long, just_right] bsdf.save(tempfilename, data1) r, e = run_local('view', tempfilename) assert not e assert too_long not in r and ('x' * 39 + u'\u2026') in r.replace( '\\u2026', u'\u2026') assert just_right in r # Test float32 for cov data1 = [3.14159, 42.0] bsdf.save(tempfilename, data1, float64=False) r, e = run_local('view', tempfilename) assert not e assert '3.14159' in r and '42.0' in r # Test unclosed stream s = bsdf.ListStream() data1 = [3, 4, 5, s] # with open(tempfilename, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) # r, e = run_local('view', tempfilename) assert not e assert '6' in r and '7' in r assert 'stream' in r and not '2' in r
def test_custom_extensions(): class MyExtension(bsdf.Extension): name = 'myob' def encode(self, s, v): assert isinstance(s, bsdf.BsdfSerializer) return v.val def decode(self, s, v): assert isinstance(s, bsdf.BsdfSerializer) return MyObject1(v) class MyExtension1(MyExtension): cls = MyObject1 def match(self, s, v): return False class MyExtension2(MyExtension): cls = MyObject1, MyObject2 def match(self, s, v): return False class MyExtension3(MyExtension): cls = MyObject1 # default: def match(self, v): return isinstance(v, self.cls) # Define data a1 = [1, MyObject1(2), 3] a2 = [1, MyObject1(2), MyObject2(3), 4] # Extension 1 can only encode MyObject1 b1 = bsdf.encode(a1, [MyExtension1]) c1 = bsdf.decode(b1, [MyExtension1]) d1 = bsdf.decode(b1, []) assert repr(a1) == repr(c1) assert repr(a1) != repr(d1) assert d1 == [1, 2, 3] # just has the undelying value # with raises(TypeError): b2 = bsdf.encode(a2, [MyExtension1]) # Extension 2 can encode both b1 = bsdf.encode(a1, [MyExtension2]) c1 = bsdf.decode(b1, [MyExtension2]) assert repr(a1) == repr(c1) # b2 = bsdf.encode(a2, [MyExtension2]) c2 = bsdf.decode(b2, [MyExtension2]) assert repr(a2).replace('ct2', 'ct1') == repr(c2) # Extension 3 can encode both too b3 = bsdf.encode(a1, [MyExtension2]) c3 = bsdf.decode(b1, [MyExtension2]) assert repr(a1) == repr(c1) # b3 = bsdf.encode(a2, [MyExtension3]) c3 = bsdf.decode(b2, [MyExtension3]) assert repr(a2).replace('ct2', 'ct1') == repr(c2) # Overwriting works b3 = bsdf.encode(a2, [MyExtension1, MyExtension3]) c3 = bsdf.decode(b2, [MyExtension1, MyExtension3]) assert repr(a2).replace('ct2', 'ct1') == repr(c2)