def test_view_random():
sys.path.insert(
0, os.path.abspath(os.path.join(__file__, '..', '..', '_tools')))
import datagen
excludes = {}
for e in os.getenv('BSDF_TEST_EXCLUDES', '').split(','):
excludes[e.strip()] = True
types = list(datagen.ALL_TYPES)
if 'ndarray' in excludes:
types.remove('ndarray')
# Process a few random dicts
for iter in range(8):
random.seed(time.time())
data1 = datagen.random_dict(6, maxn=100, types=types)
bsdf.save(tempfilename, data1)
r, e = run_local('view', tempfilename)
assert not e
assert r
# not much more to assert, except that it does not crash
# Process a few random lists
for iter in range(8):
random.seed(time.time())
data1 = datagen.random_list(6, maxn=100, types=types)
bsdf.save(tempfilename, data1)
r, e = run_local('view', tempfilename)
assert not e
assert r
def test_info():
data1 = [3, 4, 5]
bsdf.save(tempfilename, data1)
r, e = run_local('info', tempfilename)
r = r.replace(' ', ' ').replace(' ', ' ').replace(' ', ' ')
assert not e
assert 'file_size' in r
assert 'valid: true' in r
assert 'version:' in r
# Not a file
r, e = run_local('info', '~/foo_does_not_exist.bsdf')
assert not r
assert 'invalid file' in e
# Not a bsdf file
with open(tempfilename, 'wb') as f:
pass
r, e = run_local('info', tempfilename)
r = r.replace(' ', ' ').replace(' ', ' ').replace(' ', ' ')
assert not e
assert 'file_size' in r
assert 'valid: false' in r
assert 'version: ?' in r
def save(fname, data):
""" Save to json or bsdf, depending on the extension. """
if fname.endswith('.json'):
with open(fname, 'wt', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False)
elif fname.endswith('.bsdf'):
bsdf.save(fname, data)
else:
assert False
def test_streaming_closing123():
""" Writing a streamed list, closing the stream. """
for iter in range(4):
f = io.BytesIO()
thelist = bsdf.ListStream()
bsdf.save(f, thelist)
thelist.append('hi')
# closing here will write the length of the stream, marking the stream as closed
if iter in (1, 2):
thelist.close()
for i in range(3):
thelist.append(i)
if iter == 2:
thelist.close()
if iter == 3:
thelist.close(True)
assert thelist.count == 4
assert thelist.index == thelist.count
bb = f.getvalue()
b1 = bsdf.decode(bb)
b2 = bsdf.decode(bb, load_streaming=True)
if iter in (0, 1, 2):
assert isinstance(b2, bsdf.ListStream) and not isinstance(b2, list)
else:
assert not isinstance(b2, bsdf.ListStream) and isinstance(b2, list)
if iter == 0:
# Not closed
assert b1 == ['hi', 0, 1, 2]
assert list(b2) == ['hi', 0, 1, 2]
elif iter == 1:
# Closed, and elements added later
assert b1 == ['hi']
assert list(b2) == ['hi']
elif iter == 2:
# Closed twice
assert b1 == ['hi', 0, 1, 2]
assert list(b2) == ['hi', 0, 1, 2]
elif iter == 3:
# Closed twice
assert b1 == ['hi', 0, 1, 2]
assert list(b2) == ['hi', 0, 1, 2]
else:
assert False
def test_liststreaming_reading1():
""" Reading a streamed list. """
f = io.BytesIO()
thelist = bsdf.ListStream()
a = [3, 4, thelist]
bsdf.save(f, a)
thelist.append('hi')
for i in range(3):
thelist.append(i)
bb = f.getvalue()
b = bsdf.decode(bb, load_streaming=True)
x = b[-1]
assert isinstance(x, bsdf.ListStream)
assert x.next() == 'hi'
assert x.next() == 0
assert x.next() == 1
assert x.next() == 2
with raises(StopIteration):
x.next()
# Support iteration
b = bsdf.decode(bb, load_streaming=True)
x = b[-1]
xx = [i for i in x]
assert xx == ['hi', 0, 1, 2]
# Cannot read after file is closed
f = io.BytesIO(bb)
b = bsdf.load(f, load_streaming=True)
x = b[-1]
f.close()
with raises(IOError):
x.next()
# Cannot read when in read-mode
ls = bsdf.ListStream()
with raises(IOError):
ls.next()
with raises(IOError):
list(ls)
# mmm, this would never happen, I guess, but need associated file!
ls = bsdf.ListStream('r')
with raises(IOError):
ls.next()
def cmd_create(filename, code):
""" Create a BSDF file from data obtained by evaluation Python code.
Usage: bsdf filename code
examples:
bsdf foo.bsdf {meaning:42}
bsdf bar.bsdf [3,4]*100
"""
try:
data = eval(code)
except Exception as err:
raise Exception('Could not evaluate Python code to generate data:'
'\n ' + str(err))
bsdf.save(filename, data)
def cmd_convert(filename1, filename2):
""" Convert one format into another (e.g. JSON to BSDF).
Formats currently supported are: JSON, BSDF.
Note that conversion might fail (e.g. JSON does not support binary blobs).
Usage: bsdf convert filename1 filename2
File types are derived from the extensions.
"""
fname1 = filename1.lower()
fname2 = filename2.lower()
if fname1.endswith('.json'):
with open(filename1, 'rt', encoding='utf-8') as f:
data = json.load(f)
elif fname1.endswith('.bsdf'):
with open(filename1, 'rb') as f:
data = bsdf.load(f)
else:
fail('Unknown load format extension for %r' % filename1)
try:
if fname2.endswith('.json'):
okw = dict(mode='wt', encoding='utf-8')
okw = dict(mode='wb') if sys.version_info < (3, ) else okw
with open(filename2, **okw) as f:
json.dump(data, f)
elif fname2.endswith('.bsdf'):
with open(filename2, 'wb') as f:
bsdf.save(f, data)
else:
fail('Unknown save format extension for %r' % filename1)
except Exception:
try:
os.remove(filename2)
except Exception: # pragma: no cover
pass
raise
_print('Wrote', filename2)
def test_liststreaming1():
""" Writing a streamed list. """
f = io.BytesIO()
thelist = bsdf.ListStream()
a = [3, 4, thelist]
bsdf.save(f, a)
thelist.append('hi')
for i in range(10):
thelist.append(i * 101)
thelist.append((4, 2))
thelist.close()
bb = f.getvalue()
b = bsdf.decode(bb)
assert b[-1] == [
'hi', 0, 101, 202, 303, 404, 505, 606, 707, 808, 909, [4, 2]
]
def test_loaders_and_savers():
# load and save is already tested above and in many other places
s1 = dict(foo=42, bar=[1, 2.1, False, 'spam', b'eggs'])
# In-memory
bb = bsdf.encode(s1)
s2 = bsdf.decode(bb)
assert s1 == s2
# Using a filename
bsdf.save(tempfilename, s1)
s2 = bsdf.load(tempfilename)
assert s1 == s2
# Using a file object
with open(tempfilename, 'wb') as f:
bsdf.save(f, s1)
with open(tempfilename, 'rb') as f:
s2 = bsdf.load(f)
assert s1 == s2
# Using a very strict file object
with open(tempfilename, 'wb') as f:
bsdf.save(StrictWriteFile(f), s1)
with open(tempfilename, 'rb') as f:
s2 = bsdf.load(StrictReadFile(f))
assert s1 == s2
def test_convert():
tempfilename1 = tempfilename
tempfilename2 = tempfilename + '.json'
data1 = [4, 5, 6]
bsdf.save(tempfilename, data1)
# Convert to json
r, e = run_local('convert', tempfilename1, tempfilename2)
assert not e
assert tempfilename2 in r
# Convert back
r, e = run_local('convert', tempfilename2, tempfilename1)
assert not e
assert tempfilename1 in r
# Check
assert open(tempfilename2, 'rb').read().decode().strip() == '[4, 5, 6]'
data2 = bsdf.load(tempfilename)
assert data1 == data2
# Fail, unknown extension
r, e = run_local('convert', tempfilename1 + '.png', tempfilename1)
assert not r
assert 'unknown' in e.lower() and 'extension' in e.lower() and 'load' in e
#
r, e = run_local('convert', tempfilename1, tempfilename1 + '.png')
assert not r
assert 'unknown' in e.lower() and 'extension' in e.lower() and 'save' in e
if sys.version_info < (3, ):
return
# Cannot convert bytes and nan
bsdf.save(tempfilename, [bsdf.Blob(b'xx'), float('nan')])
r, e = run_local('convert', tempfilename1, tempfilename2)
assert not r
assert 'not JSON serializable' in e
assert not os.path.isfile(tempfilename2) # file was deleted/not written
def test_blob_modding3(): # actual files
bsdf.save(tempfilename, bsdf.Blob(b'xxyyzz', extra_size=2))
# Can read, but not modify in rb mode
with open(tempfilename, 'rb') as f:
blob = bsdf.load(f, lazy_blob=True)
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(4)
with raises(IOError):
blob.write(b'aa')
# But we can in a+ mode
with open(tempfilename, 'r+b') as f:
blob = bsdf.load(f, lazy_blob=True)
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(4)
blob.write(b'aa')
assert bsdf.load(tempfilename) == b'xxyyaa'
def test_liststream_modding():
# Create a file
ls = bsdf.ListStream()
with open(tempfilename, 'wb') as f:
bsdf.save(f, ls)
ls.append('foo')
ls.append('bar')
assert bsdf.load(tempfilename) == ['foo', 'bar']
# Append items hard-core more
with open(tempfilename, 'ab') as f:
f.write(b'v')
f.write(b'h*\x00') # ord('*') == 42
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42]
# Append items using the BSDF API
with open(tempfilename, 'r+b') as f:
ls = bsdf.load(f, load_streaming=True)
for i in ls:
pass
ls.append(4)
ls.append(5)
ls.close() # also close here
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5]
# Try adding more items
with open(tempfilename, 'ab') as f:
f.write(b'v')
f.write(b'h*\x00') # ord('*') == 42
# But no effect
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5]
from io import open # pypy and py27 compat
import bsdf
# Get filenames
fname1 = sys.argv[1]
fname2 = sys.argv[2]
assert os.path.isfile(fname1)
assert not os.path.isfile(fname2)
# Read data
if fname1.endswith('.json'):
with open(fname1, 'rt', encoding='utf-8') as f:
data = json.load(f)
elif fname1.endswith('.bsdf'):
data = bsdf.load(fname1)
else:
raise NotImplementedError()
# Write data
if fname2.endswith('.json'):
okw = dict(mode='wt', encoding='utf-8')
okw = dict(mode='wb') if sys.version_info < (3, ) else okw
with open(fname2, **okw) as f:
json.dump(data, f)
elif fname2.endswith('.bsdf'):
bsdf.save(fname2, data)
else:
raise NotImplementedError()
def test_bsdf_to_bsdf_extensions(**excludes):
# Test extension using complex number
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data1 = 3 + 4j
try:
bsdf.save(fname1, data1)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert isinstance(data2, complex)
assert data1 == data2
print_dot()
# Test extension using nd arrays
if 'ndarray' not in excludes:
import numpy as np
for dtype in ['uint8', 'int16', 'int32', 'float32']:
for shape in [(24, ), (24, 1), (1, 24), (4, 6), (2, 3, 4)]:
# uint8 is tricky since it may represent bytes in e.g. Matlab
if 'uint8_1d' in excludes:
if dtype == 'uint8' and len(shape) == 1 or min(shape) == 1:
continue
data1 = np.arange(24, dtype=dtype)
data1.shape = shape
try:
bsdf.save(fname1, data1)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert isinstance(data2, np.ndarray)
if shape == (24, ): # Matlab ...
assert data2.shape == (24, ) or data2.shape == (
24, 1) or data2.shape == (1, 24)
else:
assert data2.shape == shape
assert data2.dtype == dtype
assert np.all(data1 == data2)
print_dot()
# Deal with unknown extensions by leaving data through
class MyExtension(bsdf.Extension):
name = 'test.foo'
cls = threading.Thread
def encode(self, s, v):
return [7, 42]
def decode(self, s, v):
return None
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data1 = ['hi', threading.Thread(), 'there']
try:
bsdf.save(fname1, data1, [MyExtension])
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert data2 == ['hi', [7, 42], 'there']
print_dot()
def test_bsdf_to_bsdf(**excludes):
# Just repeat these
for data1 in JSON_ABLE_OBJECTS[:-1]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
print_dot()
# Singletons, some JSON implementations choke on these
for data1 in [None, False, True, 3.4, '', 'hello', [], {}, b'', b'xx']:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
assert str(data1) == str(data2), str(
(data1, data2)) # because False != 0
print_dot()
for data1 in [0, 1, 0.0, 1.0, 4]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
if 'roundfloatisnotint' not in excludes:
assert str(data1) == str(data2), str(
(data1, data2)) # because 0.0 != 0
print_dot()
# Special values
for data1 in [float('nan'), float('inf'), float('-inf')]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
#compare_data(data1, data2)
assert str(data1) == str(data2) # because nan != nan
print_dot()
# Use float32 for encoding floats
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data1 = [1, 2, 3, 4.2, 5.6, 6.001]
try:
bsdf.save(fname1, data1, float64=False)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert data1 != data2
assert all([(abs(d1 - d2) < 0.001) for d1, d2 in zip(data1, data2)])
print_dot()
# Test bytes / blobs
# We do not test compression in shared tests, since its not a strict requirement
for data1 in [
[3, 4, b'', b'x', b'yyy', 5],
[
5, 6,
bsdf.Blob(b'foo', compression=0, extra_size=20,
use_checksum=False), 7
],
[
5, 6,
bsdf.Blob(b'foo', compression=0, extra_size=10, use_checksum=True),
7
],
]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
# Compare, but turn blobs into bytes
data1 = [
x.get_bytes() if isinstance(x, bsdf.Blob) else x for x in data1
]
compare_data(data1, data2)
print_dot()
# Test alignment
for i in range(9):
data1 = ['x' * i, b'd'] # ord('d') == 100
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
try:
save(fname1, data1)
invoke_runner(fname1, fname2)
raw2 = open(fname2, 'rb').read()
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
index = raw2.find(100)
assert index % 8 == 0
# Test stream 1 (unclosed)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.append(8)
s.append(9)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7, 8, 9]]
print_dot()
# Test stream 2 (closed early)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close()
s.append(8)
s.append(9)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7]]
print_dot()
# Test stream 3 (closed twice)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close()
s.append(8)
s.append(9)
s.close()
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7, 8, 9]]
print_dot()
# Test stream 4 (close hard)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close(True)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7]]
print_dot()
def test_view():
# Create file
data1 = [1, 2, 3, [4, 4, 4, 4, 4], 8, 9]
bsdf.save(tempfilename, data1)
# Test content, plain
r, e = run_local('view', tempfilename)
assert not e
assert tempfilename not in r
assert r.count('4') == 5
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, plus info
r, e = run_local('view', tempfilename, '--info')
assert not e
assert tempfilename in r
assert 'file_size:' in r
assert r.count('4') >= 5 # can also occur in meta info
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, max depth 1
r, e = run_local('view', tempfilename, '--depth=1')
assert not e
assert r.count('4') == 0 # collapsed
assert '5' in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Test content, max depth 0
r, e = run_local('view', tempfilename, '--depth=0')
assert not e
assert r.count('\n') == 1 # all collapsed
assert '5' not in r # number of elements in inner list
assert '6' in r # number of elements in outer list
# Fail - not a file
r, e = run_local('view', '~/foo_does_not_exist.bsdf')
assert not r
assert 'invalid file' in e
# Fail - not a bsdf file
with open(tempfilename, 'wb') as f:
pass
r, e = run_local('view', tempfilename)
assert not r
assert 'does not look like a BSDF file' in e
# Fail - major version is off
bb = bsdf.encode(data1)
with open(tempfilename, 'wb') as f:
f.write(bb[:4] + b'\x00' + bb[5:])
r, e = run_local('view', tempfilename)
assert r == '' and 'different major version' in e
# Warn - minor version is lower than file
bb = bsdf.encode(data1)
with open(tempfilename, 'wb') as f:
f.write(bb[:5] + b'\xff' + bb[6:])
r, e = run_local('view', tempfilename)
assert not e
assert 'higher minor version' in r
assert r.count('4') >= 5
# Test string truncation
too_long = u'x' * 200
just_right = u'x' + '_' * 38 + 'x'
data1 = [too_long, just_right]
bsdf.save(tempfilename, data1)
r, e = run_local('view', tempfilename)
assert not e
assert too_long not in r and ('x' * 39 + u'\u2026') in r.replace(
'\\u2026', u'\u2026')
assert just_right in r
# Test float32 for cov
data1 = [3.14159, 42.0]
bsdf.save(tempfilename, data1, float64=False)
r, e = run_local('view', tempfilename)
assert not e
assert '3.14159' in r and '42.0' in r
# Test unclosed stream
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
with open(tempfilename, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
#
r, e = run_local('view', tempfilename)
assert not e
assert '6' in r and '7' in r
assert 'stream' in r and not '2' in r
def save_bsdf(filename, object, encode=True):
''' save object to a BSDF file. optionally BSDF-encoded to bytes. '''
if encode:
object = bsdf.encode(object) # type: bytes
bsdf.save(filename, object)
def test_liststreaming_write_fails():
# Only write mode
f = io.BytesIO()
ls = bsdf.ListStream('r')
with raises(ValueError):
bsdf.save(f, ls)
with raises(IOError):
ls.append(3)
with raises(IOError):
ls.close()
# Dont append (or close) before use
ls = bsdf.ListStream()
with raises(IOError):
ls.append(3)
with raises(IOError):
ls.close()
# Only use once!
f = io.BytesIO()
ls = bsdf.ListStream()
bsdf.save(f, ls)
with raises(IOError):
bsdf.save(f, ls)
# Do not write when closed!
f = io.BytesIO()
ls = bsdf.ListStream()
bsdf.save(f, ls)
f.close()
with raises(IOError):
ls.append(3)
with raises(IOError):
ls.close()
# Only ListStream
class MyStream(bsdf.BaseStream):
pass
f = io.BytesIO()
a = [3, 4, MyStream()]
with raises(TypeError):
bsdf.save(f, a)
# Only one!
f = io.BytesIO()
a = [3, 4, bsdf.ListStream(), bsdf.ListStream()]
with raises(ValueError):
bsdf.save(f, a)
# Stream must be at the end
f = io.BytesIO()
a = [3, 4, bsdf.ListStream(), 5]
with raises(ValueError):
bsdf.save(f, a)