def test_compressor_as_filter(): for compressor in compressors: if compressor is None: # skip continue print(repr(compressor)) # setup filters dtype = 'i8' filters = [ Delta(dtype=dtype), compressor ] # setup data and arrays data = np.arange(10000, dtype=dtype) a1 = array(data, chunks=1000, compressor=None, filters=filters) a2 = array(data, chunks=1000, compressor=compressor, filters=filters[:1]) # check storage for i in range(10): x = buffer_tobytes(a1.store[str(i)]) y = buffer_tobytes(a2.store[str(i)]) eq(x, y) # check data assert_array_equal(data, a1[:]) assert_array_equal(a1[:], a2[:])
def _test_encode(self, arr, **kwargs): # setup codec = self.init_codec(**kwargs) # encoding should support any object exporting the buffer protocol, # as well as array.array in PY2 # test encoding of numpy array buf = arr enc = codec.encode(buf) enc_bytes = buffer_tobytes(enc) # test encoding of raw bytes buf = arr.tobytes(order='A') enc = codec.encode(buf) actual = buffer_tobytes(enc) eq(enc_bytes, actual) # test encoding of array.array buf = array.array('b', arr.tobytes(order='A')) enc = codec.encode(buf) actual = buffer_tobytes(enc) eq(enc_bytes, actual)
def _test_decode_lossy(self, arr, decimal, **kwargs): if arr.flags.f_contiguous: order = 'F' else: order = 'C' # setup codec = self.init_codec(**kwargs) # encode enc = codec.encode(arr) enc_bytes = buffer_tobytes(enc) # decoding should support any object exporting the buffer protocol, # as well as array.array in PY2 # test decoding of raw bytes buf = enc_bytes dec = codec.decode(buf) dec = np.frombuffer(dec, dtype=arr.dtype).reshape(arr.shape, order=order) assert_array_almost_equal(arr, dec, decimal=decimal) # test decoding of array.array buf = array.array('b', enc_bytes) dec = codec.decode(buf) dec = np.frombuffer(dec, dtype=arr.dtype).reshape(arr.shape, order=order) assert_array_almost_equal(arr, dec, decimal=decimal) # test decoding of numpy array buf = np.frombuffer(enc_bytes, dtype='u1') dec = codec.decode(buf) dec = np.frombuffer(dec, dtype=arr.dtype).reshape(arr.shape, order=order) assert_array_almost_equal(arr, dec, decimal=decimal) # test decoding into numpy array out = np.empty_like(arr) codec.decode(enc_bytes, out=out) assert_array_almost_equal(arr, out, decimal=decimal) # test decoding into bytearray out = bytearray(arr.nbytes) codec.decode(enc_bytes, out=out) out = np.frombuffer(out, dtype=arr.dtype).reshape(arr.shape, order=order) assert_array_almost_equal(arr, out, decimal=decimal)