def test_liststreaming_reading1(): """ Reading a streamed list. """ f = io.BytesIO() thelist = bsdf.ListStream() a = [3, 4, thelist] bsdf.save(f, a) thelist.append('hi') for i in range(3): thelist.append(i) bb = f.getvalue() b = bsdf.decode(bb, load_streaming=True) x = b[-1] assert isinstance(x, bsdf.ListStream) assert x.next() == 'hi' assert x.next() == 0 assert x.next() == 1 assert x.next() == 2 with raises(StopIteration): x.next() # Support iteration b = bsdf.decode(bb, load_streaming=True) x = b[-1] xx = [i for i in x] assert xx == ['hi', 0, 1, 2] # Cannot read after file is closed f = io.BytesIO(bb) b = bsdf.load(f, load_streaming=True) x = b[-1] f.close() with raises(IOError): x.next() # Cannot read when in read-mode ls = bsdf.ListStream() with raises(IOError): ls.next() with raises(IOError): list(ls) # mmm, this would never happen, I guess, but need associated file! ls = bsdf.ListStream('r') with raises(IOError): ls.next()
def test_liststreaming_write_fails(): # Only write mode f = io.BytesIO() ls = bsdf.ListStream('r') with raises(ValueError): bsdf.save(f, ls) with raises(IOError): ls.append(3) with raises(IOError): ls.close() # Dont append (or close) before use ls = bsdf.ListStream() with raises(IOError): ls.append(3) with raises(IOError): ls.close() # Only use once! f = io.BytesIO() ls = bsdf.ListStream() bsdf.save(f, ls) with raises(IOError): bsdf.save(f, ls) # Do not write when closed! f = io.BytesIO() ls = bsdf.ListStream() bsdf.save(f, ls) f.close() with raises(IOError): ls.append(3) with raises(IOError): ls.close() # Only ListStream class MyStream(bsdf.BaseStream): pass f = io.BytesIO() a = [3, 4, MyStream()] with raises(TypeError): bsdf.save(f, a) # Only one! f = io.BytesIO() a = [3, 4, bsdf.ListStream(), bsdf.ListStream()] with raises(ValueError): bsdf.save(f, a) # Stream must be at the end f = io.BytesIO() a = [3, 4, bsdf.ListStream(), 5] with raises(ValueError): bsdf.save(f, a)
def test_streaming_closing123(): """ Writing a streamed list, closing the stream. """ for iter in range(4): f = io.BytesIO() thelist = bsdf.ListStream() bsdf.save(f, thelist) thelist.append('hi') # closing here will write the length of the stream, marking the stream as closed if iter in (1, 2): thelist.close() for i in range(3): thelist.append(i) if iter == 2: thelist.close() if iter == 3: thelist.close(True) assert thelist.count == 4 assert thelist.index == thelist.count bb = f.getvalue() b1 = bsdf.decode(bb) b2 = bsdf.decode(bb, load_streaming=True) if iter in (0, 1, 2): assert isinstance(b2, bsdf.ListStream) and not isinstance(b2, list) else: assert not isinstance(b2, bsdf.ListStream) and isinstance(b2, list) if iter == 0: # Not closed assert b1 == ['hi', 0, 1, 2] assert list(b2) == ['hi', 0, 1, 2] elif iter == 1: # Closed, and elements added later assert b1 == ['hi'] assert list(b2) == ['hi'] elif iter == 2: # Closed twice assert b1 == ['hi', 0, 1, 2] assert list(b2) == ['hi', 0, 1, 2] elif iter == 3: # Closed twice assert b1 == ['hi', 0, 1, 2] assert list(b2) == ['hi', 0, 1, 2] else: assert False
def test_liststreaming1(): """ Writing a streamed list. """ f = io.BytesIO() thelist = bsdf.ListStream() a = [3, 4, thelist] bsdf.save(f, a) thelist.append('hi') for i in range(10): thelist.append(i * 101) thelist.append((4, 2)) thelist.close() bb = f.getvalue() b = bsdf.decode(bb) assert b[-1] == [ 'hi', 0, 101, 202, 303, 404, 505, 606, 707, 808, 909, [4, 2] ]
def test_liststream_modding(): # Create a file ls = bsdf.ListStream() with open(tempfilename, 'wb') as f: bsdf.save(f, ls) ls.append('foo') ls.append('bar') assert bsdf.load(tempfilename) == ['foo', 'bar'] # Append items hard-core more with open(tempfilename, 'ab') as f: f.write(b'v') f.write(b'h*\x00') # ord('*') == 42 assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42] # Append items using the BSDF API with open(tempfilename, 'r+b') as f: ls = bsdf.load(f, load_streaming=True) for i in ls: pass ls.append(4) ls.append(5) ls.close() # also close here assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5] # Try adding more items with open(tempfilename, 'ab') as f: f.write(b'v') f.write(b'h*\x00') # ord('*') == 42 # But no effect assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5]
def test_bsdf_to_bsdf(**excludes): # Just repeat these for data1 in JSON_ABLE_OBJECTS[:-1]: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) compare_data(data1, data2) print_dot() # Singletons, some JSON implementations choke on these for data1 in [None, False, True, 3.4, '', 'hello', [], {}, b'', b'xx']: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) compare_data(data1, data2) assert str(data1) == str(data2), str( (data1, data2)) # because False != 0 print_dot() for data1 in [0, 1, 0.0, 1.0, 4]: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) compare_data(data1, data2) if 'roundfloatisnotint' not in excludes: assert str(data1) == str(data2), str( (data1, data2)) # because 0.0 != 0 print_dot() # Special values for data1 in [float('nan'), float('inf'), float('-inf')]: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) #compare_data(data1, data2) assert str(data1) == str(data2) # because nan != nan print_dot() # Use float32 for encoding floats fname1, fname2 = get_filenames('.bsdf', '.bsdf') data1 = [1, 2, 3, 4.2, 5.6, 6.001] try: bsdf.save(fname1, data1, float64=False) invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) except Exception: print(data1) raise finally: remove(fname1, fname2) assert data1 != data2 assert all([(abs(d1 - d2) < 0.001) for d1, d2 in zip(data1, data2)]) print_dot() # Test bytes / blobs # We do not test compression in shared tests, since its not a strict requirement for data1 in [ [3, 4, b'', b'x', b'yyy', 5], [ 5, 6, bsdf.Blob(b'foo', compression=0, extra_size=20, use_checksum=False), 7 ], [ 5, 6, bsdf.Blob(b'foo', compression=0, extra_size=10, use_checksum=True), 7 ], ]: fname1, fname2 = get_filenames('.bsdf', '.bsdf') data2 = convert_data(fname1, fname2, data1) # Compare, but turn blobs into bytes data1 = [ x.get_bytes() if isinstance(x, bsdf.Blob) else x for x in data1 ] compare_data(data1, data2) print_dot() # Test alignment for i in range(9): data1 = ['x' * i, b'd'] # ord('d') == 100 fname1, fname2 = get_filenames('.bsdf', '.bsdf') try: save(fname1, data1) invoke_runner(fname1, fname2) raw2 = open(fname2, 'rb').read() except Exception: print(data1) raise finally: remove(fname1, fname2) index = raw2.find(100) assert index % 8 == 0 # Test stream 1 (unclosed) s = bsdf.ListStream() data1 = [3, 4, 5, s] # fname1, fname2 = get_filenames('.bsdf', '.bsdf') with open(fname1, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) s.append(8) s.append(9) invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) assert data2 == [3, 4, 5, [6, 7, 8, 9]] print_dot() # Test stream 2 (closed early) s = bsdf.ListStream() data1 = [3, 4, 5, s] # fname1, fname2 = get_filenames('.bsdf', '.bsdf') with open(fname1, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) s.close() s.append(8) s.append(9) invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) assert data2 == [3, 4, 5, [6, 7]] print_dot() # Test stream 3 (closed twice) s = bsdf.ListStream() data1 = [3, 4, 5, s] # fname1, fname2 = get_filenames('.bsdf', '.bsdf') with open(fname1, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) s.close() s.append(8) s.append(9) s.close() invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) assert data2 == [3, 4, 5, [6, 7, 8, 9]] print_dot() # Test stream 4 (close hard) s = bsdf.ListStream() data1 = [3, 4, 5, s] # fname1, fname2 = get_filenames('.bsdf', '.bsdf') with open(fname1, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) s.close(True) invoke_runner(fname1, fname2) data2 = bsdf.load(fname2) assert data2 == [3, 4, 5, [6, 7]] print_dot()
def test_view(): # Create file data1 = [1, 2, 3, [4, 4, 4, 4, 4], 8, 9] bsdf.save(tempfilename, data1) # Test content, plain r, e = run_local('view', tempfilename) assert not e assert tempfilename not in r assert r.count('4') == 5 assert '5' in r # number of elements in inner list assert '6' in r # number of elements in outer list # Test content, plus info r, e = run_local('view', tempfilename, '--info') assert not e assert tempfilename in r assert 'file_size:' in r assert r.count('4') >= 5 # can also occur in meta info assert '5' in r # number of elements in inner list assert '6' in r # number of elements in outer list # Test content, max depth 1 r, e = run_local('view', tempfilename, '--depth=1') assert not e assert r.count('4') == 0 # collapsed assert '5' in r # number of elements in inner list assert '6' in r # number of elements in outer list # Test content, max depth 0 r, e = run_local('view', tempfilename, '--depth=0') assert not e assert r.count('\n') == 1 # all collapsed assert '5' not in r # number of elements in inner list assert '6' in r # number of elements in outer list # Fail - not a file r, e = run_local('view', '~/foo_does_not_exist.bsdf') assert not r assert 'invalid file' in e # Fail - not a bsdf file with open(tempfilename, 'wb') as f: pass r, e = run_local('view', tempfilename) assert not r assert 'does not look like a BSDF file' in e # Fail - major version is off bb = bsdf.encode(data1) with open(tempfilename, 'wb') as f: f.write(bb[:4] + b'\x00' + bb[5:]) r, e = run_local('view', tempfilename) assert r == '' and 'different major version' in e # Warn - minor version is lower than file bb = bsdf.encode(data1) with open(tempfilename, 'wb') as f: f.write(bb[:5] + b'\xff' + bb[6:]) r, e = run_local('view', tempfilename) assert not e assert 'higher minor version' in r assert r.count('4') >= 5 # Test string truncation too_long = u'x' * 200 just_right = u'x' + '_' * 38 + 'x' data1 = [too_long, just_right] bsdf.save(tempfilename, data1) r, e = run_local('view', tempfilename) assert not e assert too_long not in r and ('x' * 39 + u'\u2026') in r.replace( '\\u2026', u'\u2026') assert just_right in r # Test float32 for cov data1 = [3.14159, 42.0] bsdf.save(tempfilename, data1, float64=False) r, e = run_local('view', tempfilename) assert not e assert '3.14159' in r and '42.0' in r # Test unclosed stream s = bsdf.ListStream() data1 = [3, 4, 5, s] # with open(tempfilename, 'wb') as f: bsdf.save(f, data1) s.append(6) s.append(7) # r, e = run_local('view', tempfilename) assert not e assert '6' in r and '7' in r assert 'stream' in r and not '2' in r