def test_blob_writing1(): # Use blob to specify bytes blob = bsdf.Blob(b'xxxx') bb1 = bsdf.encode([2, 3, blob, 5]) # Again, with extra size blob = bsdf.Blob(b'xxxx', extra_size=100) bb2 = bsdf.encode([2, 3, blob, 5]) # Again, with checksum blob = bsdf.Blob(b'xxxx', use_checksum=True) bb3 = bsdf.encode([2, 3, blob, 5]) assert len(bb2) == len(bb1) + 100 assert len(bb3) == len(bb1) + 16 assert bsdf.decode(bb1) == [2, 3, b'xxxx', 5] assert bsdf.decode(bb2) == [2, 3, b'xxxx', 5] assert bsdf.decode(bb3) == [2, 3, b'xxxx', 5] # Fail with raises(TypeError): bsdf.Blob([1, 2, 3]) with raises(RuntimeError): blob.tell() with raises(RuntimeError): blob.seek(0) with raises(RuntimeError): blob.read(1) with raises(RuntimeError): blob.write(b'xx')
def test_blob_reading2(): bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2)) f = io.BytesIO(bb) blob = bsdf.load(f, lazy_blob=True) # Always seek first blob.seek(0) assert blob.read(2) == b'xx' assert blob.tell() == 2 assert blob.read(2) == b'yy' # We can overwrite, but changes an internal file that we cannot use :P blob.write(b'aa') blob.seek(0) assert blob.read(2) == b'xx' blob.seek(-2) # relative to allocated size assert blob.tell() == 6 # can just about do this, due to extra size blob.seek(8) # But this is too far with raises(IOError): blob.seek(9) # And so is this blob.seek(6) with raises(IOError): blob.write(b'xxx') # And this blob.seek(6) with raises(IOError): blob.read(3)
def test_blob_modding2(): # with checksum bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2, use_checksum=True)) f = io.BytesIO(bb) blob = bsdf.load(f, lazy_blob=True) blob.seek(4) blob.write(b'aa') blob.update_checksum() assert bsdf.decode(f.getvalue()) == b'xxyyaa'
def test_blob_reading3(): # compression # ZLib bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=1)) f = io.BytesIO(bb) blob = bsdf.load(f, lazy_blob=True) # blob.get_bytes() == b'xxyyzz' # BZ2 bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=2)) f = io.BytesIO(bb) blob = bsdf.load(f, lazy_blob=True) # blob.get_bytes() == b'xxyyzz' # But we cannot read or write blob.seek(0) with raises(IOError): blob.read(2) with raises(IOError): blob.write(b'aa')
def test_blob_reading1(): blob = bsdf.Blob(b'xxxx') bb1 = bsdf.encode([2, 3, blob, 5]) res1 = bsdf.decode(bb1) assert isinstance(res1[2], bytes) res1 = bsdf.decode(bb1, lazy_blob=True) assert not isinstance(res1[2], bytes) and isinstance(res1[2], bsdf.Blob) res1[2].get_bytes() == b'xxxx'
def test_compression(): # Compressing makes smaller files data = [1, 2, b'\x00' * 10000] b1 = bsdf.encode(data, compression=0) b2 = bsdf.encode(data, compression=1) b3 = bsdf.encode(data, compression=2) assert len(b1) > 10 * len(b2) assert len(b1) > 10 * len(b3) # Compression can be per-object, using blobs data1 = [1, 2, b'\x00' * 10000] data2 = [1, 2, bsdf.Blob(b'\x00' * 10000, compression=1)] b1 = bsdf.encode(data1, compression=0) b2 = bsdf.encode(data2, compression=0) assert len(b1) > 10 * len(b2)
def test_convert(): tempfilename1 = tempfilename tempfilename2 = tempfilename + '.json' data1 = [4, 5, 6] bsdf.save(tempfilename, data1) # Convert to json r, e = run_local('convert', tempfilename1, tempfilename2) assert not e assert tempfilename2 in r # Convert back r, e = run_local('convert', tempfilename2, tempfilename1) assert not e assert tempfilename1 in r # Check assert open(tempfilename2, 'rb').read().decode().strip() == '[4, 5, 6]' data2 = bsdf.load(tempfilename) assert data1 == data2 # Fail, unknown extension r, e = run_local('convert', tempfilename1 + '.png', tempfilename1) assert not r assert 'unknown' in e.lower() and 'extension' in e.lower() and 'load' in e # r, e = run_local('convert', tempfilename1, tempfilename1 + '.png') assert not r assert 'unknown' in e.lower() and 'extension' in e.lower() and 'save' in e if sys.version_info < (3, ): return # Cannot convert bytes and nan bsdf.save(tempfilename, [bsdf.Blob(b'xx'), float('nan')]) r, e = run_local('convert', tempfilename1, tempfilename2) assert not r assert 'not JSON serializable' in e assert not os.path.isfile(tempfilename2) # file was deleted/not written
def test_blob_modding3(): # actual files bsdf.save(tempfilename, bsdf.Blob(b'xxyyzz', extra_size=2)) # Can read, but not modify in rb mode with open(tempfilename, 'rb') as f: blob = bsdf.load(f, lazy_blob=True) blob.seek(0) assert blob.read(2) == b'xx' blob.seek(4) with raises(IOError): blob.write(b'aa') # But we can in a+ mode with open(tempfilename, 'r+b') as f: blob = bsdf.load(f, lazy_blob=True) blob.seek(0) assert blob.read(2) == b'xx' blob.seek(4) blob.write(b'aa') assert bsdf.load(tempfilename) == b'xxyyaa'
def test_bsdf_to_bsdf(**excludes): # Just repeat these for data1 in JSON_ABLE_OBJECTS[:-1]: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) compare_data(data1, data2) print_dot() # Singletons, some JSON implementations choke on these for data1 in [None, False, True, 3.4, '', 'hello', [], {}, b'', b'xx']: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) compare_data(data1, data2) assert str(data1) == str(data2), str( (data1, data2)) # because False != 0 print_dot() for data1 in [0, 1, 0.0, 1.0, 4]: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) compare_data(data1, data2) if 'roundfloatisnotint' not in excludes: assert str(data1) == str(data2), str( (data1, data2)) # because 0.0 != 0 print_dot() # Special values for data1 in [float('nan'), float('inf'), float('-inf')]: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) #compare_data(data1, data2) assert str(data1) == str(data2) # because nan != nan print_dot() # Use float32 for encoding floats fname1, fname2 = get_filenames('.bsdf', '.bsdf') data1 = [1, 2, 3, 4.2, 5.6, 6.001] try: bsdf.save(fname1, data1, float64=False) invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) except Exception: print(data1) raise finally: remove(fname1, fname2) assert data1 != data2 assert all([(abs(d1 - d2) < 0.001) for d1, d2 in zip(data1, data2)]) print_dot() # Test bytes / blobs # We do not test compression in shared tests, since its not a strict requirement for data1 in [ [3, 4, b'', b'x', b'yyy', 5], [ 5, 6, bsdf.Blob(b'foo', compression=0, extra_size=20, use_checksum=False), 7 ], [ 5, 6, bsdf.Blob(b'foo', compression=0, extra_size=10, use_checksum=True), 7 ], ]: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) # Compare, but turn blobs into bytes data1 = [ x.get_bytes() if isinstance(x, bsdf.Blob) else x for x in data1 ] compare_data(data1, data2) print_dot() # Test alignment for i in range(9): data1 = ['x' * i, b'd'] # ord('d') == 100 fname1, fname2 = get_filenames('.bsdf', '.bsdf') try: save(fname1, data1) invoke_runner(fname1, fname2) raw2 = open(fname2, 'rb').read() except Exception: print(data1) raise finally: remove(fname1, fname2) index = raw2.find(100) assert index % 8 == 0 # Test stream 1 (unclosed) s = bsdf.ListStream() data1 = [3, 4, 5, s] # fname1, fname2 = get_filenames('.bsdf', '.bsdf') with open(fname1, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) s.append(8) s.append(9) invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) assert data2 == [3, 4, 5, [6, 7, 8, 9]] print_dot() # Test stream 2 (closed early) s = bsdf.ListStream() data1 = [3, 4, 5, s] # fname1, fname2 = get_filenames('.bsdf', '.bsdf') with open(fname1, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) s.close() s.append(8) s.append(9) invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) assert data2 == [3, 4, 5, [6, 7]] print_dot() # Test stream 3 (closed twice) s = bsdf.ListStream() data1 = [3, 4, 5, s] # fname1, fname2 = get_filenames('.bsdf', '.bsdf') with open(fname1, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) s.close() s.append(8) s.append(9) s.close() invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) assert data2 == [3, 4, 5, [6, 7, 8, 9]] print_dot() # Test stream 4 (close hard) s = bsdf.ListStream() data1 = [3, 4, 5, s] # fname1, fname2 = get_filenames('.bsdf', '.bsdf') with open(fname1, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) s.close(True) invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) assert data2 == [3, 4, 5, [6, 7]] print_dot()