def test_liststreaming_reading1():
""" Reading a streamed list. """
f = io.BytesIO()
thelist = bsdf.ListStream()
a = [3, 4, thelist]
bsdf.save(f, a)
thelist.append('hi')
for i in range(3):
thelist.append(i)
bb = f.getvalue()
b = bsdf.decode(bb, load_streaming=True)
x = b[-1]
assert isinstance(x, bsdf.ListStream)
assert x.next() == 'hi'
assert x.next() == 0
assert x.next() == 1
assert x.next() == 2
with raises(StopIteration):
x.next()
# Support iteration
b = bsdf.decode(bb, load_streaming=True)
x = b[-1]
xx = [i for i in x]
assert xx == ['hi', 0, 1, 2]
# Cannot read after file is closed
f = io.BytesIO(bb)
b = bsdf.load(f, load_streaming=True)
x = b[-1]
f.close()
with raises(IOError):
x.next()
# Cannot read when in read-mode
ls = bsdf.ListStream()
with raises(IOError):
ls.next()
with raises(IOError):
list(ls)
# mmm, this would never happen, I guess, but need associated file!
ls = bsdf.ListStream('r')
with raises(IOError):
ls.next()
def test_liststreaming_write_fails():
# Only write mode
f = io.BytesIO()
ls = bsdf.ListStream('r')
with raises(ValueError):
bsdf.save(f, ls)
with raises(IOError):
ls.append(3)
with raises(IOError):
ls.close()
# Dont append (or close) before use
ls = bsdf.ListStream()
with raises(IOError):
ls.append(3)
with raises(IOError):
ls.close()
# Only use once!
f = io.BytesIO()
ls = bsdf.ListStream()
bsdf.save(f, ls)
with raises(IOError):
bsdf.save(f, ls)
# Do not write when closed!
f = io.BytesIO()
ls = bsdf.ListStream()
bsdf.save(f, ls)
f.close()
with raises(IOError):
ls.append(3)
with raises(IOError):
ls.close()
# Only ListStream
class MyStream(bsdf.BaseStream):
pass
f = io.BytesIO()
a = [3, 4, MyStream()]
with raises(TypeError):
bsdf.save(f, a)
# Only one!
f = io.BytesIO()
a = [3, 4, bsdf.ListStream(), bsdf.ListStream()]
with raises(ValueError):
bsdf.save(f, a)
# Stream must be at the end
f = io.BytesIO()
a = [3, 4, bsdf.ListStream(), 5]
with raises(ValueError):
bsdf.save(f, a)
def test_streaming_closing123():
""" Writing a streamed list, closing the stream. """
for iter in range(4):
f = io.BytesIO()
thelist = bsdf.ListStream()
bsdf.save(f, thelist)
thelist.append('hi')
# closing here will write the length of the stream, marking the stream as closed
if iter in (1, 2):
thelist.close()
for i in range(3):
thelist.append(i)
if iter == 2:
thelist.close()
if iter == 3:
thelist.close(True)
assert thelist.count == 4
assert thelist.index == thelist.count
bb = f.getvalue()
b1 = bsdf.decode(bb)
b2 = bsdf.decode(bb, load_streaming=True)
if iter in (0, 1, 2):
assert isinstance(b2, bsdf.ListStream) and not isinstance(b2, list)
else:
assert not isinstance(b2, bsdf.ListStream) and isinstance(b2, list)
if iter == 0:
# Not closed
assert b1 == ['hi', 0, 1, 2]
assert list(b2) == ['hi', 0, 1, 2]
elif iter == 1:
# Closed, and elements added later
assert b1 == ['hi']
assert list(b2) == ['hi']
elif iter == 2:
# Closed twice
assert b1 == ['hi', 0, 1, 2]
assert list(b2) == ['hi', 0, 1, 2]
elif iter == 3:
# Closed twice
assert b1 == ['hi', 0, 1, 2]
assert list(b2) == ['hi', 0, 1, 2]
else:
assert False
def test_liststreaming1():
""" Writing a streamed list. """
f = io.BytesIO()
thelist = bsdf.ListStream()
a = [3, 4, thelist]
bsdf.save(f, a)
thelist.append('hi')
for i in range(10):
thelist.append(i * 101)
thelist.append((4, 2))
thelist.close()
bb = f.getvalue()
b = bsdf.decode(bb)
assert b[-1] == [
'hi', 0, 101, 202, 303, 404, 505, 606, 707, 808, 909, [4, 2]
]
def test_liststream_modding():
# Create a file
ls = bsdf.ListStream()
with open(tempfilename, 'wb') as f:
bsdf.save(f, ls)
ls.append('foo')
ls.append('bar')
assert bsdf.load(tempfilename) == ['foo', 'bar']
# Append items hard-core more
with open(tempfilename, 'ab') as f:
f.write(b'v')
f.write(b'h*\x00') # ord('*') == 42
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42]
# Append items using the BSDF API
with open(tempfilename, 'r+b') as f:
ls = bsdf.load(f, load_streaming=True)
for i in ls:
pass
ls.append(4)
ls.append(5)
ls.close() # also close here
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5]
# Try adding more items
with open(tempfilename, 'ab') as f:
f.write(b'v')
f.write(b'h*\x00') # ord('*') == 42
# But no effect
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5]
def test_bsdf_to_bsdf(**excludes):
# Just repeat these
for data1 in JSON_ABLE_OBJECTS[:-1]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
print_dot()
# Singletons, some JSON implementations choke on these
for data1 in [None, False, True, 3.4, '', 'hello', [], {}, b'', b'xx']:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
assert str(data1) == str(data2), str(
(data1, data2)) # because False != 0
print_dot()
for data1 in [0, 1, 0.0, 1.0, 4]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
if 'roundfloatisnotint' not in excludes:
assert str(data1) == str(data2), str(
(data1, data2)) # because 0.0 != 0
print_dot()
# Special values
for data1 in [float('nan'), float('inf'), float('-inf')]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
#compare_data(data1, data2)
assert str(data1) == str(data2) # because nan != nan
print_dot()
# Use float32 for encoding floats
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data1 = [1, 2, 3, 4.2, 5.6, 6.001]
try:
bsdf.save(fname1, data1, float64=False)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert data1 != data2
assert all([(abs(d1 - d2) < 0.001) for d1, d2 in zip(data1, data2)])
print_dot()
# Test bytes / blobs
# We do not test compression in shared tests, since its not a strict requirement
for data1 in [
[3, 4, b'', b'x', b'yyy', 5],
[
5, 6,
bsdf.Blob(b'foo', compression=0, extra_size=20,
use_checksum=False), 7
],
[
5, 6,
bsdf.Blob(b'foo', compression=0, extra_size=10, use_checksum=True),
7
],
]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
# Compare, but turn blobs into bytes
data1 = [
x.get_bytes() if isinstance(x, bsdf.Blob) else x for x in data1
]
compare_data(data1, data2)
print_dot()
# Test alignment
for i in range(9):
data1 = ['x' * i, b'd'] # ord('d') == 100
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
try:
save(fname1, data1)
invoke_runner(fname1, fname2)
raw2 = open(fname2, 'rb').read()
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
index = raw2.find(100)
assert index % 8 == 0
# Test stream 1 (unclosed)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.append(8)
s.append(9)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7, 8, 9]]
print_dot()
# Test stream 2 (closed early)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close()
s.append(8)
s.append(9)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7]]
print_dot()
# Test stream 3 (closed twice)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close()
s.append(8)
s.append(9)
s.close()
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7, 8, 9]]
print_dot()
# Test stream 4 (close hard)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close(True)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7]]
print_dot()
def test_view():
# Create file
data1 = [1, 2, 3, [4, 4, 4, 4, 4], 8, 9]
bsdf.save(tempfilename, data1)
# Test content, plain
r, e = run_local('view', tempfilename)
assert not e
assert tempfilename not in r
assert r.count('4') == 5
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, plus info
r, e = run_local('view', tempfilename, '--info')
assert not e
assert tempfilename in r
assert 'file_size:' in r
assert r.count('4') >= 5 # can also occur in meta info
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, max depth 1
r, e = run_local('view', tempfilename, '--depth=1')
assert not e
assert r.count('4') == 0 # collapsed
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, max depth 0
r, e = run_local('view', tempfilename, '--depth=0')
assert not e
assert r.count('\n') == 1 # all collapsed
assert '5' not in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Fail - not a file
r, e = run_local('view', '~/foo_does_not_exist.bsdf')
assert not r
assert 'invalid file' in e
# Fail - not a bsdf file
with open(tempfilename, 'wb') as f:
pass
r, e = run_local('view', tempfilename)
assert not r
assert 'does not look like a BSDF file' in e
# Fail - major version is off
bb = bsdf.encode(data1)
with open(tempfilename, 'wb') as f:
f.write(bb[:4] + b'\x00' + bb[5:])
r, e = run_local('view', tempfilename)
assert r == '' and 'different major version' in e
# Warn - minor version is lower than file
bb = bsdf.encode(data1)
with open(tempfilename, 'wb') as f:
f.write(bb[:5] + b'\xff' + bb[6:])
r, e = run_local('view', tempfilename)
assert not e
assert 'higher minor version' in r
assert r.count('4') >= 5
# Test string truncation
too_long = u'x' * 200
just_right = u'x' + '_' * 38 + 'x'
data1 = [too_long, just_right]
bsdf.save(tempfilename, data1)
r, e = run_local('view', tempfilename)
assert not e
assert too_long not in r and ('x' * 39 + u'\u2026') in r.replace(
'\\u2026', u'\u2026')
assert just_right in r
# Test float32 for cov
data1 = [3.14159, 42.0]
bsdf.save(tempfilename, data1, float64=False)
r, e = run_local('view', tempfilename)
assert not e
assert '3.14159' in r and '42.0' in r
# Test unclosed stream
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
with open(tempfilename, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
#
r, e = run_local('view', tempfilename)
assert not e
assert '6' in r and '7' in r
assert 'stream' in r and not '2' in r