def test_array_with_delta_filter():
# setup
astype = 'u1'
dtype = 'i8'
filters = [Delta(astype=astype, dtype=dtype)]
data = np.arange(100, dtype=dtype)
for compressor in compressors:
a = array(data, chunks=10, compressor=compressor, filters=filters)
# check round-trip
assert_array_equal(data, a[:])
# check chunks
for i in range(10):
cdata = a.store[str(i)]
if compressor:
chunk = compressor.decode(cdata)
else:
chunk = cdata
actual = np.frombuffer(chunk, dtype=astype)
expect = np.array([i * 10] + ([1] * 9), dtype=astype)
assert_array_equal(expect, actual)
def test_compressor_as_filter():
for compressor in compressors:
if compressor is None:
# skip
continue
# setup filters
dtype = 'i8'
filters = [
Delta(dtype=dtype),
compressor
]
# setup data and arrays
data = np.arange(10000, dtype=dtype)
a1 = array(data, chunks=1000, compressor=None, filters=filters)
a2 = array(data, chunks=1000, compressor=compressor,
filters=filters[:1])
# check storage
for i in range(10):
x = bytes(a1.store[str(i)])
y = bytes(a2.store[str(i)])
assert x == y
# check data
assert_array_equal(data, a1[:])
assert_array_equal(a1[:], a2[:])
def create_array(read_only=False, **kwargs):
store = dict()
dtype = kwargs.get('dtype', None)
filters = [
Delta(dtype=dtype),
FixedScaleOffset(dtype=dtype, scale=1, offset=0),
]
kwargs.setdefault('filters', filters)
compressor = Zlib(1)
kwargs.setdefault('compressor', compressor)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def testStoreZarrExecution(self):
raw = np.random.RandomState(0).rand(10, 20)
group_name = 'test_group'
dataset_name = 'test_dataset'
t = tensor(raw, chunk_size=6)
with self.assertRaises(TypeError):
tozarr(object(), t)
with tempfile.TemporaryDirectory() as d:
filename = os.path.join(
d, 'test_store_{}.zarr'.format(int(time.time())))
path = '{}/{}/{}'.format(filename, group_name, dataset_name)
r = tozarr(filename,
t,
group=group_name,
dataset=dataset_name,
compressor=Zstd(level=3))
self.executor.execute_tensor(r)
arr = zarr.open(path)
np.testing.assert_array_equal(arr, raw)
self.assertEqual(arr.compressor, Zstd(level=3))
r = tozarr(path, t + 2)
self.executor.execute_tensor(r)
arr = zarr.open(path)
np.testing.assert_array_equal(arr, raw + 2)
filters = [Delta(dtype='i4')]
compressor = Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE)
arr = zarr.open(path, compressor=compressor, filters=filters)
r = tozarr(arr, t + 1)
self.executor.execute_tensor(r)
result = zarr.open_array(path)
np.testing.assert_array_equal(result, raw + 1)
def test_store_zarr_execution(setup):
raw = np.random.RandomState(0).rand(10, 20)
group_name = 'test_group'
dataset_name = 'test_dataset'
t = tensor(raw, chunk_size=6)
with pytest.raises(TypeError):
tozarr(object(), t)
with tempfile.TemporaryDirectory() as d:
filename = os.path.join(d, f'test_store_{int(time.time())}.zarr')
path = f'{filename}/{group_name}/{dataset_name}'
r = tozarr(filename,
t,
group=group_name,
dataset=dataset_name,
compressor=Zstd(level=3))
r.execute()
arr = zarr.open(path)
np.testing.assert_array_equal(arr, raw)
assert arr.compressor == Zstd(level=3)
r = tozarr(path, t + 2)
r.execute()
arr = zarr.open(path)
np.testing.assert_array_equal(arr, raw + 2)
filters = [Delta(dtype='i4')]
compressor = Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE)
arr = zarr.open(path, compressor=compressor, filters=filters)
r = tozarr(arr, t + 1)
r.execute()
result = zarr.open_array(path)
np.testing.assert_array_equal(result, raw + 1)
def test_init_accessor_twice_raises():
ds = xr.Dataset({'foo': (['x'], [1, 2, 3])})
ds.rest(app_kws={'foo': 'bar'})
with pytest.raises(RuntimeError) as excinfo:
ds.rest(app_kws={'bar': 'foo'})
excinfo.match(r'This accessor has already been initialized')
@pytest.mark.parametrize(
'filters, compressor',
[
(None, None),
(None, Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE)),
([Delta(dtype='i4')
], Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE)),
],
)
def test_encode_chunk(filters, compressor):
buf = np.arange(10).tobytes()
ebuf = encode_chunk(buf, filters=filters, compressor=compressor)
assert isinstance(ebuf, bytes)
def test_encode_object_array_raises():
buf = np.arange(10).astype('O')
with pytest.raises(RuntimeError):
encode_chunk(buf)