def test_compressor_as_filter():
for compressor in compressors:
if compressor is None:
# skip
continue
print(repr(compressor))
# setup filters
dtype = 'i8'
filters = [
Delta(dtype=dtype),
compressor
]
# setup data and arrays
data = np.arange(10000, dtype=dtype)
a1 = array(data, chunks=1000, compressor=None, filters=filters)
a2 = array(data, chunks=1000, compressor=compressor,
filters=filters[:1])
# check storage
for i in range(10):
x = buffer_tobytes(a1.store[str(i)])
y = buffer_tobytes(a2.store[str(i)])
eq(x, y)
# check data
assert_array_equal(data, a1[:])
assert_array_equal(a1[:], a2[:])
def _test_encode(self, arr, **kwargs):
# setup
codec = self.init_codec(**kwargs)
# encoding should support any object exporting the buffer protocol,
# as well as array.array in PY2
# test encoding of numpy array
buf = arr
enc = codec.encode(buf)
enc_bytes = buffer_tobytes(enc)
# test encoding of raw bytes
buf = arr.tobytes(order='A')
enc = codec.encode(buf)
actual = buffer_tobytes(enc)
eq(enc_bytes, actual)
# test encoding of array.array
buf = array.array('b', arr.tobytes(order='A'))
enc = codec.encode(buf)
actual = buffer_tobytes(enc)
eq(enc_bytes, actual)
def _test_encode(self, arr, **kwargs):
# setup
codec = self.init_codec(**kwargs)
# encoding should support any object exporting the buffer protocol,
# as well as array.array in PY2
# test encoding of numpy array
buf = arr
enc = codec.encode(buf)
enc_bytes = buffer_tobytes(enc)
# test encoding of raw bytes
buf = arr.tobytes(order='A')
enc = codec.encode(buf)
actual = buffer_tobytes(enc)
eq(enc_bytes, actual)
# test encoding of array.array
buf = array.array('b', arr.tobytes(order='A'))
enc = codec.encode(buf)
actual = buffer_tobytes(enc)
eq(enc_bytes, actual)
def _test_decode_lossy(self, arr, decimal, **kwargs):
if arr.flags.f_contiguous:
order = 'F'
else:
order = 'C'
# setup
codec = self.init_codec(**kwargs)
# encode
enc = codec.encode(arr)
enc_bytes = buffer_tobytes(enc)
# decoding should support any object exporting the buffer protocol,
# as well as array.array in PY2
# test decoding of raw bytes
buf = enc_bytes
dec = codec.decode(buf)
dec = np.frombuffer(dec, dtype=arr.dtype).reshape(arr.shape,
order=order)
assert_array_almost_equal(arr, dec, decimal=decimal)
# test decoding of array.array
buf = array.array('b', enc_bytes)
dec = codec.decode(buf)
dec = np.frombuffer(dec, dtype=arr.dtype).reshape(arr.shape,
order=order)
assert_array_almost_equal(arr, dec, decimal=decimal)
# test decoding of numpy array
buf = np.frombuffer(enc_bytes, dtype='u1')
dec = codec.decode(buf)
dec = np.frombuffer(dec, dtype=arr.dtype).reshape(arr.shape,
order=order)
assert_array_almost_equal(arr, dec, decimal=decimal)
# test decoding into numpy array
out = np.empty_like(arr)
codec.decode(enc_bytes, out=out)
assert_array_almost_equal(arr, out, decimal=decimal)
# test decoding into bytearray
out = bytearray(arr.nbytes)
codec.decode(enc_bytes, out=out)
out = np.frombuffer(out, dtype=arr.dtype).reshape(arr.shape,
order=order)
assert_array_almost_equal(arr, out, decimal=decimal)
def _test_decode_lossy(self, arr, decimal, **kwargs):
if arr.flags.f_contiguous:
order = 'F'
else:
order = 'C'
# setup
codec = self.init_codec(**kwargs)
# encode
enc = codec.encode(arr)
enc_bytes = buffer_tobytes(enc)
# decoding should support any object exporting the buffer protocol,
# as well as array.array in PY2
# test decoding of raw bytes
buf = enc_bytes
dec = codec.decode(buf)
dec = np.frombuffer(dec, dtype=arr.dtype).reshape(arr.shape,
order=order)
assert_array_almost_equal(arr, dec, decimal=decimal)
# test decoding of array.array
buf = array.array('b', enc_bytes)
dec = codec.decode(buf)
dec = np.frombuffer(dec, dtype=arr.dtype).reshape(arr.shape,
order=order)
assert_array_almost_equal(arr, dec, decimal=decimal)
# test decoding of numpy array
buf = np.frombuffer(enc_bytes, dtype='u1')
dec = codec.decode(buf)
dec = np.frombuffer(dec, dtype=arr.dtype).reshape(arr.shape,
order=order)
assert_array_almost_equal(arr, dec, decimal=decimal)
# test decoding into numpy array
out = np.empty_like(arr)
codec.decode(enc_bytes, out=out)
assert_array_almost_equal(arr, out, decimal=decimal)
# test decoding into bytearray
out = bytearray(arr.nbytes)
codec.decode(enc_bytes, out=out)
out = np.frombuffer(out, dtype=arr.dtype).reshape(arr.shape,
order=order)
assert_array_almost_equal(arr, out, decimal=decimal)
