def test_blob_writing1():
# Use blob to specify bytes
blob = bsdf.Blob(b'xxxx')
bb1 = bsdf.encode([2, 3, blob, 5])
# Again, with extra size
blob = bsdf.Blob(b'xxxx', extra_size=100)
bb2 = bsdf.encode([2, 3, blob, 5])
# Again, with checksum
blob = bsdf.Blob(b'xxxx', use_checksum=True)
bb3 = bsdf.encode([2, 3, blob, 5])
assert len(bb2) == len(bb1) + 100
assert len(bb3) == len(bb1) + 16
assert bsdf.decode(bb1) == [2, 3, b'xxxx', 5]
assert bsdf.decode(bb2) == [2, 3, b'xxxx', 5]
assert bsdf.decode(bb3) == [2, 3, b'xxxx', 5]
# Fail
with raises(TypeError):
bsdf.Blob([1, 2, 3])
with raises(RuntimeError):
blob.tell()
with raises(RuntimeError):
blob.seek(0)
with raises(RuntimeError):
blob.read(1)
with raises(RuntimeError):
blob.write(b'xx')
def test_blob_reading2():
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
# Always seek first
blob.seek(0)
assert blob.read(2) == b'xx'
assert blob.tell() == 2
assert blob.read(2) == b'yy'
# We can overwrite, but changes an internal file that we cannot use :P
blob.write(b'aa')
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(-2) # relative to allocated size
assert blob.tell() == 6
# can just about do this, due to extra size
blob.seek(8)
# But this is too far
with raises(IOError):
blob.seek(9)
# And so is this
blob.seek(6)
with raises(IOError):
blob.write(b'xxx')
# And this
blob.seek(6)
with raises(IOError):
blob.read(3)
def test_blob_modding2(): # with checksum
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', extra_size=2, use_checksum=True))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
blob.seek(4)
blob.write(b'aa')
blob.update_checksum()
assert bsdf.decode(f.getvalue()) == b'xxyyaa'
def test_blob_reading3(): # compression
# ZLib
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=1))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
#
blob.get_bytes() == b'xxyyzz'
# BZ2
bb = bsdf.encode(bsdf.Blob(b'xxyyzz', compression=2))
f = io.BytesIO(bb)
blob = bsdf.load(f, lazy_blob=True)
#
blob.get_bytes() == b'xxyyzz'
# But we cannot read or write
blob.seek(0)
with raises(IOError):
blob.read(2)
with raises(IOError):
blob.write(b'aa')
def test_blob_reading1():
blob = bsdf.Blob(b'xxxx')
bb1 = bsdf.encode([2, 3, blob, 5])
res1 = bsdf.decode(bb1)
assert isinstance(res1[2], bytes)
res1 = bsdf.decode(bb1, lazy_blob=True)
assert not isinstance(res1[2], bytes) and isinstance(res1[2], bsdf.Blob)
res1[2].get_bytes() == b'xxxx'
def test_compression():
# Compressing makes smaller files
data = [1, 2, b'\x00' * 10000]
b1 = bsdf.encode(data, compression=0)
b2 = bsdf.encode(data, compression=1)
b3 = bsdf.encode(data, compression=2)
assert len(b1) > 10 * len(b2)
assert len(b1) > 10 * len(b3)
# Compression can be per-object, using blobs
data1 = [1, 2, b'\x00' * 10000]
data2 = [1, 2, bsdf.Blob(b'\x00' * 10000, compression=1)]
b1 = bsdf.encode(data1, compression=0)
b2 = bsdf.encode(data2, compression=0)
assert len(b1) > 10 * len(b2)
def test_convert():
tempfilename1 = tempfilename
tempfilename2 = tempfilename + '.json'
data1 = [4, 5, 6]
bsdf.save(tempfilename, data1)
# Convert to json
r, e = run_local('convert', tempfilename1, tempfilename2)
assert not e
assert tempfilename2 in r
# Convert back
r, e = run_local('convert', tempfilename2, tempfilename1)
assert not e
assert tempfilename1 in r
# Check
assert open(tempfilename2, 'rb').read().decode().strip() == '[4, 5, 6]'
data2 = bsdf.load(tempfilename)
assert data1 == data2
# Fail, unknown extension
r, e = run_local('convert', tempfilename1 + '.png', tempfilename1)
assert not r
assert 'unknown' in e.lower() and 'extension' in e.lower() and 'load' in e
#
r, e = run_local('convert', tempfilename1, tempfilename1 + '.png')
assert not r
assert 'unknown' in e.lower() and 'extension' in e.lower() and 'save' in e
if sys.version_info < (3, ):
return
# Cannot convert bytes and nan
bsdf.save(tempfilename, [bsdf.Blob(b'xx'), float('nan')])
r, e = run_local('convert', tempfilename1, tempfilename2)
assert not r
assert 'not JSON serializable' in e
assert not os.path.isfile(tempfilename2) # file was deleted/not written
def test_blob_modding3(): # actual files
bsdf.save(tempfilename, bsdf.Blob(b'xxyyzz', extra_size=2))
# Can read, but not modify in rb mode
with open(tempfilename, 'rb') as f:
blob = bsdf.load(f, lazy_blob=True)
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(4)
with raises(IOError):
blob.write(b'aa')
# But we can in a+ mode
with open(tempfilename, 'r+b') as f:
blob = bsdf.load(f, lazy_blob=True)
blob.seek(0)
assert blob.read(2) == b'xx'
blob.seek(4)
blob.write(b'aa')
assert bsdf.load(tempfilename) == b'xxyyaa'
def test_bsdf_to_bsdf(**excludes):
# Just repeat these
for data1 in JSON_ABLE_OBJECTS[:-1]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
print_dot()
# Singletons, some JSON implementations choke on these
for data1 in [None, False, True, 3.4, '', 'hello', [], {}, b'', b'xx']:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
assert str(data1) == str(data2), str(
(data1, data2)) # because False != 0
print_dot()
for data1 in [0, 1, 0.0, 1.0, 4]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
compare_data(data1, data2)
if 'roundfloatisnotint' not in excludes:
assert str(data1) == str(data2), str(
(data1, data2)) # because 0.0 != 0
print_dot()
# Special values
for data1 in [float('nan'), float('inf'), float('-inf')]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
#compare_data(data1, data2)
assert str(data1) == str(data2) # because nan != nan
print_dot()
# Use float32 for encoding floats
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data1 = [1, 2, 3, 4.2, 5.6, 6.001]
try:
bsdf.save(fname1, data1, float64=False)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
assert data1 != data2
assert all([(abs(d1 - d2) < 0.001) for d1, d2 in zip(data1, data2)])
print_dot()
# Test bytes / blobs
# We do not test compression in shared tests, since its not a strict requirement
for data1 in [
[3, 4, b'', b'x', b'yyy', 5],
[
5, 6,
bsdf.Blob(b'foo', compression=0, extra_size=20,
use_checksum=False), 7
],
[
5, 6,
bsdf.Blob(b'foo', compression=0, extra_size=10, use_checksum=True),
7
],
]:
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
data2 = convert_data(fname1, fname2, data1)
# Compare, but turn blobs into bytes
data1 = [
x.get_bytes() if isinstance(x, bsdf.Blob) else x for x in data1
]
compare_data(data1, data2)
print_dot()
# Test alignment
for i in range(9):
data1 = ['x' * i, b'd'] # ord('d') == 100
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
try:
save(fname1, data1)
invoke_runner(fname1, fname2)
raw2 = open(fname2, 'rb').read()
except Exception:
print(data1)
raise
finally:
remove(fname1, fname2)
index = raw2.find(100)
assert index % 8 == 0
# Test stream 1 (unclosed)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.append(8)
s.append(9)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7, 8, 9]]
print_dot()
# Test stream 2 (closed early)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close()
s.append(8)
s.append(9)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7]]
print_dot()
# Test stream 3 (closed twice)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close()
s.append(8)
s.append(9)
s.close()
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7, 8, 9]]
print_dot()
# Test stream 4 (close hard)
s = bsdf.ListStream()
data1 = [3, 4, 5, s]
#
fname1, fname2 = get_filenames('.bsdf', '.bsdf')
with open(fname1, 'wb') as f:
bsdf.save(f, data1)
s.append(6)
s.append(7)
s.close(True)
invoke_runner(fname1, fname2)
data2 = bsdf.load(fname2)
assert data2 == [3, 4, 5, [6, 7]]
print_dot()