def test_format_compatibility():
# This test is intended to catch any unintended changes that break the ability to
# read data stored with a previous minor version (which should be format-compatible).
# fixture data
fixture = group(store=DirectoryStore('fixture'))
# set seed to get consistent random data
np.random.seed(42)
arrays_chunks = [
(np.arange(1111, dtype='i1'), 100),
(np.arange(1111, dtype='i2'), 100),
(np.arange(1111, dtype='i4'), 100),
(np.arange(1111, dtype='i8'), 1000),
(np.random.randint(0, 200, size=2222, dtype='u1'), 100),
(np.random.randint(0, 2000, size=2222, dtype='u2'), 100),
(np.random.randint(0, 2000, size=2222, dtype='u4'), 100),
(np.random.randint(0, 2000, size=2222, dtype='u8'), 100),
(np.linspace(0, 1, 3333, dtype='f2'), 100),
(np.linspace(0, 1, 3333, dtype='f4'), 100),
(np.linspace(0, 1, 3333, dtype='f8'), 100),
(np.random.normal(loc=0, scale=1, size=4444).astype('f2'), 100),
(np.random.normal(loc=0, scale=1, size=4444).astype('f4'), 100),
(np.random.normal(loc=0, scale=1, size=4444).astype('f8'), 100),
(np.random.choice([b'A', b'C', b'G', b'T'], size=5555,
replace=True).astype('S'), 100),
(np.random.choice(['foo', 'bar', 'baz', 'quux'],
size=5555,
replace=True).astype('U'), 100),
(np.random.choice([0, 1 / 3, 1 / 7, 1 / 9, np.nan],
size=5555,
replace=True).astype('f8'), 100),
(np.random.randint(0, 2, size=5555, dtype=bool), 100),
(np.arange(20000, dtype='i4').reshape(2000, 10, order='C'), (100, 3)),
(np.arange(20000, dtype='i4').reshape(200, 100, order='F'), (100, 30)),
(np.arange(20000, dtype='i4').reshape(200, 10, 10,
order='C'), (100, 3, 3)),
(np.arange(20000, dtype='i4').reshape(20, 100, 10,
order='F'), (10, 30, 3)),
(np.arange(20000, dtype='i4').reshape(20, 10, 10, 10,
order='C'), (10, 3, 3, 3)),
(np.arange(20000, dtype='i4').reshape(20, 10, 10, 10,
order='F'), (10, 3, 3, 3)),
]
compressors = [
None,
Zlib(level=1),
BZ2(level=1),
Blosc(cname='zstd', clevel=1, shuffle=0),
Blosc(cname='zstd', clevel=1, shuffle=1),
Blosc(cname='zstd', clevel=1, shuffle=2),
Blosc(cname='lz4', clevel=1, shuffle=0),
]
for i, (arr, chunks) in enumerate(arrays_chunks):
if arr.flags.f_contiguous:
order = 'F'
else:
order = 'C'
for j, compressor in enumerate(compressors):
path = '{}/{}'.format(i, j)
if path not in fixture: # pragma: no cover
# store the data - should be one-time operation
fixture.array(path,
data=arr,
chunks=chunks,
order=order,
compressor=compressor)
# setup array
z = fixture[path]
# check contents
if arr.dtype.kind == 'f':
assert_array_almost_equal(arr, z[:])
else:
assert_array_equal(arr, z[:])
# check dtype
eq(arr.dtype, z.dtype)
# check compressor
if compressor is None:
assert_is_none(z.compressor)
else:
eq(compressor.codec_id, z.compressor.codec_id)
eq(compressor.get_config(), z.compressor.get_config())
def create_store(self):
path = tempfile.mkdtemp()
atexit.register(atexit_rmtree, path)
store = DirectoryStore(path)
return store