def testArrowSerialize(self): try: import numpy as np from numpy.testing import assert_array_equal except ImportError: np = None try: import scipy.sparse as sps except ImportError: sps = None if np: array = np.random.rand(1000, 100) assert_array_equal( array, dataserializer.deserialize( dataserializer.serialize(array).to_buffer())) if sps: mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr')) des_mat = dataserializer.deserialize( dataserializer.serialize(mat).to_buffer()) self.assertTrue((mat.spmatrix != des_mat.spmatrix).nnz == 0) if np and sps: array = np.random.rand(1000, 100) mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr')) tp = (array, mat) des_tp = dataserializer.deserialize( dataserializer.serialize(tp).to_buffer()) assert_array_equal(tp[0], des_tp[0]) self.assertTrue((tp[1].spmatrix != des_tp[1].spmatrix).nnz == 0)
def testArrowSerialize(self): array = np.random.rand(1000, 100) assert_array_equal(array, dataserializer.deserialize(dataserializer.serialize(array).to_buffer())) if sps: mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr')) des_mat = dataserializer.deserialize(dataserializer.serialize(mat).to_buffer()) self.assertTrue((mat.spmatrix != des_mat.spmatrix).nnz == 0) array = np.random.rand(1000, 100) mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr')) tp = (array, mat) des_tp = dataserializer.deserialize(dataserializer.serialize(tp).to_buffer()) assert_array_equal(tp[0], des_tp[0]) self.assertTrue((tp[1].spmatrix != des_tp[1].spmatrix).nnz == 0)
def _read_data_batch(reader): bio = BytesIO() with reader: while True: buf = reader.read(io_size) if buf: bio.write(buf) else: break return dataserializer.deserialize(bio.getvalue())
def _read_data(reader): with reader: return dataserializer.deserialize(reader.read())
def _read_data(reader, idx): with reader: data_store[idx] = dataserializer.deserialize( reader.read())