def test_loaders_and_savers():
# load and save is already tested above and in many other places
s1 = dict(foo=42, bar=[1, 2.1, False, 'spam', b'eggs'])
# In-memory
bb = bsdf.encode(s1)
s2 = bsdf.decode(bb)
assert s1 == s2
# Using a filename
bsdf.save(tempfilename, s1)
s2 = bsdf.load(tempfilename)
assert s1 == s2
# Using a file object
with open(tempfilename, 'wb') as f:
bsdf.save(f, s1)
with open(tempfilename, 'rb') as f:
s2 = bsdf.load(f)
assert s1 == s2
# Using a very strict file object
with open(tempfilename, 'wb') as f:
bsdf.save(StrictWriteFile(f), s1)
with open(tempfilename, 'rb') as f:
s2 = bsdf.load(StrictReadFile(f))
assert s1 == s2
def load_bsdf(filename, decode=True):
''' load object from a BSDF file. automatically decode if in bytes. '''
object = bsdf.load(filename)
if decode and (type(object) is bytes):
return bsdf.decode(object)
else:
return object
def test_blob_reading2():
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
# Always seek first
blob.seek(0)
assert blob.read(2) == b'xx'
assert blob.tell() == 2
assert blob.read(2) == b'yy'
# We can overwrite, but changes an internal file that we cannot use :P
blob.write(b'aa')
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(-2) # relative to allocated size
assert blob.tell() == 6
# can just about do this, due to extra size
blob.seek(8)
# But this is too far
with raises(IOError):
blob.seek(9)
# And so is this
blob.seek(6)
with raises(IOError):
blob.write(b'xxx')
# And this
blob.seek(6)
with raises(IOError):
blob.read(3)
def load(fname):
""" Load from json or bsdf, depending on the extension. """
if fname.endswith('.json'):
with open(fname, 'rt', encoding='utf-8') as f:
return json.load(f)
elif fname.endswith('.bsdf'):
return bsdf.load(fname)
else:
assert False
def test_create():
r, e = run_local('create', tempfilename, '[3,4,5]*10')
data = bsdf.load(tempfilename)
assert data == [3, 4, 5] * 10
# Fail
r, e = run_local('create', tempfilename, '1/0')
assert not r
assert 'could not evaluate' in e.lower()
def test_blob_modding2(): # with checksum
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2, use_checksum=True))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
blob.seek(4)
blob.write(b'aa')
blob.update_checksum()
assert bsdf.decode(f.getvalue()) == b'xxyyaa'
def test_blob_reading3(): # compression
# ZLib
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=1))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
#
blob.get_bytes() == b'xxyyzz'
# BZ2
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=2))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
#
blob.get_bytes() == b'xxyyzz'
# But we cannot read or write
blob.seek(0)
with raises(IOError):
blob.read(2)
with raises(IOError):
blob.write(b'aa')
def test_liststreaming_reading1():
""" Reading a streamed list. """
f = io.BytesIO()
thelist = bsdf.ListStream()
a = [3, 4, thelist]
bsdf.save(f, a)
thelist.append('hi')
for i in range(3):
thelist.append(i)
bb = f.getvalue()
b = bsdf.decode(bb, load_streaming=True)
x = b[-1]
assert isinstance(x, bsdf.ListStream)
assert x.next() == 'hi'
assert x.next() == 0
assert x.next() == 1
assert x.next() == 2
with raises(StopIteration):
x.next()
# Support iteration
b = bsdf.decode(bb, load_streaming=True)
x = b[-1]
xx = [i for i in x]
assert xx == ['hi', 0, 1, 2]
# Cannot read after file is closed
f = io.BytesIO(bb)
b = bsdf.load(f, load_streaming=True)
x = b[-1]
f.close()
with raises(IOError):
x.next()
# Cannot read when in read-mode
ls = bsdf.ListStream()
with raises(IOError):
ls.next()
with raises(IOError):
list(ls)
# mmm, this would never happen, I guess, but need associated file!
ls = bsdf.ListStream('r')
with raises(IOError):
ls.next()
def test_blob_modding3(): # actual files
bsdf.save(tempfilename, bsdf.Blob(b'xxyyzz', extra_size=2))
# Can read, but not modify in rb mode
with open(tempfilename, 'rb') as f:
blob = bsdf.load(f, lazy_blob=True)
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(4)
with raises(IOError):
blob.write(b'aa')
# But we can in a+ mode
with open(tempfilename, 'r+b') as f:
blob = bsdf.load(f, lazy_blob=True)
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(4)
blob.write(b'aa')
assert bsdf.load(tempfilename) == b'xxyyaa'
def test_liststream_modding():
# Create a file
ls = bsdf.ListStream()
with open(tempfilename, 'wb') as f:
bsdf.save(f, ls)
ls.append('foo')
ls.append('bar')
assert bsdf.load(tempfilename) == ['foo', 'bar']
# Append items hard-core more
with open(tempfilename, 'ab') as f:
f.write(b'v')
f.write(b'h*\x00') # ord('*') == 42
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42]
# Append items using the BSDF API
with open(tempfilename, 'r+b') as f:
ls = bsdf.load(f, load_streaming=True)
for i in ls:
pass
ls.append(4)
ls.append(5)
ls.close() # also close here
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5]
# Try adding more items
with open(tempfilename, 'ab') as f:
f.write(b'v')
f.write(b'h*\x00') # ord('*') == 42
# But no effect
assert bsdf.load(tempfilename) == ['foo', 'bar', None, 42, 4, 5]
def cmd_convert(filename1, filename2):
""" Convert one format into another (e.g. JSON to BSDF).
Formats currently supported are: JSON, BSDF.
Note that conversion might fail (e.g. JSON does not support binary blobs).
Usage: bsdf convert filename1 filename2
File types are derived from the extensions.
"""
fname1 = filename1.lower()
fname2 = filename2.lower()
if fname1.endswith('.json'):
with open(filename1, 'rt', encoding='utf-8') as f:
data = json.load(f)
elif fname1.endswith('.bsdf'):
with open(filename1, 'rb') as f:
data = bsdf.load(f)
else:
fail('Unknown load format extension for %r' % filename1)
try:
if fname2.endswith('.json'):
okw = dict(mode='wt', encoding='utf-8')
okw = dict(mode='wb') if sys.version_info < (3, ) else okw
with open(filename2, **okw) as f:
json.dump(data, f)
elif fname2.endswith('.bsdf'):
with open(filename2, 'wb') as f:
bsdf.save(f, data)
else:
fail('Unknown save format extension for %r' % filename1)
except Exception:
try:
os.remove(filename2)
except Exception: # pragma: no cover
pass
raise
_print('Wrote', filename2)
def test_convert():
tempfilename1 = tempfilename
tempfilename2 = tempfilename + '.json'
data1 = [4, 5, 6]
bsdf.save(tempfilename, data1)
# Convert to json
r, e = run_local('convert', tempfilename1, tempfilename2)
assert not e
assert tempfilename2 in r
# Convert back
r, e = run_local('convert', tempfilename2, tempfilename1)
assert not e
assert tempfilename1 in r
# Check
assert open(tempfilename2, 'rb').read().decode().strip() == '[4, 5, 6]'
data2 = bsdf.load(tempfilename)
assert data1 == data2
# Fail, unknown extension
r, e = run_local('convert', tempfilename1 + '.png', tempfilename1)
assert not r
assert 'unknown' in e.lower() and 'extension' in e.lower() and 'load' in e
#
r, e = run_local('convert', tempfilename1, tempfilename1 + '.png')
assert not r
assert 'unknown' in e.lower() and 'extension' in e.lower() and 'save' in e
if sys.version_info < (3, ):
return
# Cannot convert bytes and nan
bsdf.save(tempfilename, [bsdf.Blob(b'xx'), float('nan')])
r, e = run_local('convert', tempfilename1, tempfilename2)
assert not r
assert 'not JSON serializable' in e
assert not os.path.isfile(tempfilename2) # file was deleted/not written
from io import open # pypy and py27 compat
import bsdf
# Get filenames
fname1 = sys.argv[1]
fname2 = sys.argv[2]
assert os.path.isfile(fname1)
assert not os.path.isfile(fname2)
# Read data
if fname1.endswith('.json'):
with open(fname1, 'rt', encoding='utf-8') as f:
data = json.load(f)
elif fname1.endswith('.bsdf'):
data = bsdf.load(fname1)
else:
raise NotImplementedError()
# Write data
if fname2.endswith('.json'):
okw = dict(mode='wt', encoding='utf-8')
okw = dict(mode='wb') if sys.version_info < (3, ) else okw
with open(fname2, **okw) as f:
json.dump(data, f)
elif fname2.endswith('.bsdf'):
bsdf.save(fname2, data)
else:
raise NotImplementedError()
def test_bsdf_to_bsdf_extensions(**excludes):
# Test extension using complex number
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data1 = 3 + 4j
try:
bsdf.save(fname1, data1)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert isinstance(data2, complex)
assert data1 == data2
print_dot()
# Test extension using nd arrays
if 'ndarray' not in excludes:
import numpy as np
for dtype in ['uint8', 'int16', 'int32', 'float32']:
for shape in [(24, ), (24, 1), (1, 24), (4, 6), (2, 3, 4)]:
# uint8 is tricky since it may represent bytes in e.g. Matlab
if 'uint8_1d' in excludes:
if dtype == 'uint8' and len(shape) == 1 or min(shape) == 1:
continue
data1 = np.arange(24, dtype=dtype)
data1.shape = shape
try:
bsdf.save(fname1, data1)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert isinstance(data2, np.ndarray)
if shape == (24, ): # Matlab ...
assert data2.shape == (24, ) or data2.shape == (
24, 1) or data2.shape == (1, 24)
else:
assert data2.shape == shape
assert data2.dtype == dtype
assert np.all(data1 == data2)
print_dot()
# Deal with unknown extensions by leaving data through
class MyExtension(bsdf.Extension):
name = 'test.foo'
cls = threading.Thread
def encode(self, s, v):
return [7, 42]
def decode(self, s, v):
return None
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data1 = ['hi', threading.Thread(), 'there']
try:
bsdf.save(fname1, data1, [MyExtension])
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert data2 == ['hi', [7, 42], 'there']
print_dot()
def test_bsdf_to_bsdf(**excludes):
# Just repeat these
for data1 in JSON_ABLE_OBJECTS[:-1]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
print_dot()
# Singletons, some JSON implementations choke on these
for data1 in [None, False, True, 3.4, '', 'hello', [], {}, b'', b'xx']:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
assert str(data1) == str(data2), str(
(data1, data2)) # because False != 0
print_dot()
for data1 in [0, 1, 0.0, 1.0, 4]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
if 'roundfloatisnotint' not in excludes:
assert str(data1) == str(data2), str(
(data1, data2)) # because 0.0 != 0
print_dot()
# Special values
for data1 in [float('nan'), float('inf'), float('-inf')]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
#compare_data(data1, data2)
assert str(data1) == str(data2) # because nan != nan
print_dot()
# Use float32 for encoding floats
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data1 = [1, 2, 3, 4.2, 5.6, 6.001]
try:
bsdf.save(fname1, data1, float64=False)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert data1 != data2
assert all([(abs(d1 - d2) < 0.001) for d1, d2 in zip(data1, data2)])
print_dot()
# Test bytes / blobs
# We do not test compression in shared tests, since its not a strict requirement
for data1 in [
[3, 4, b'', b'x', b'yyy', 5],
[
5, 6,
bsdf.Blob(b'foo', compression=0, extra_size=20,
use_checksum=False), 7
],
[
5, 6,
bsdf.Blob(b'foo', compression=0, extra_size=10, use_checksum=True),
7
],
]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
# Compare, but turn blobs into bytes
data1 = [
x.get_bytes() if isinstance(x, bsdf.Blob) else x for x in data1
]
compare_data(data1, data2)
print_dot()
# Test alignment
for i in range(9):
data1 = ['x' * i, b'd'] # ord('d') == 100
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
try:
save(fname1, data1)
invoke_runner(fname1, fname2)
raw2 = open(fname2, 'rb').read()
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
index = raw2.find(100)
assert index % 8 == 0
# Test stream 1 (unclosed)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.append(8)
s.append(9)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7, 8, 9]]
print_dot()
# Test stream 2 (closed early)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close()
s.append(8)
s.append(9)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7]]
print_dot()
# Test stream 3 (closed twice)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close()
s.append(8)
s.append(9)
s.close()
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7, 8, 9]]
print_dot()
# Test stream 4 (close hard)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close(True)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7]]
print_dot()