def testArrowSerialize(self):
try:
import numpy as np
from numpy.testing import assert_array_equal
except ImportError:
np = None
try:
import scipy.sparse as sps
except ImportError:
sps = None
if np:
array = np.random.rand(1000, 100)
assert_array_equal(
array,
dataserializer.deserialize(
dataserializer.serialize(array).to_buffer()))
if sps:
mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr'))
des_mat = dataserializer.deserialize(
dataserializer.serialize(mat).to_buffer())
self.assertTrue((mat.spmatrix != des_mat.spmatrix).nnz == 0)
if np and sps:
array = np.random.rand(1000, 100)
mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr'))
tp = (array, mat)
des_tp = dataserializer.deserialize(
dataserializer.serialize(tp).to_buffer())
assert_array_equal(tp[0], des_tp[0])
self.assertTrue((tp[1].spmatrix != des_tp[1].spmatrix).nnz == 0)
def testArrowSerialize(self):
array = np.random.rand(1000, 100)
assert_array_equal(array, dataserializer.deserialize(dataserializer.serialize(array).to_buffer()))
if sps:
mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr'))
des_mat = dataserializer.deserialize(dataserializer.serialize(mat).to_buffer())
self.assertTrue((mat.spmatrix != des_mat.spmatrix).nnz == 0)
array = np.random.rand(1000, 100)
mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr'))
tp = (array, mat)
des_tp = dataserializer.deserialize(dataserializer.serialize(tp).to_buffer())
assert_array_equal(tp[0], des_tp[0])
self.assertTrue((tp[1].spmatrix != des_tp[1].spmatrix).nnz == 0)
def _read_data_batch(reader):
bio = BytesIO()
with reader:
while True:
buf = reader.read(io_size)
if buf:
bio.write(buf)
else:
break
return dataserializer.deserialize(bio.getvalue())
def _read_data(reader):
with reader:
return dataserializer.deserialize(reader.read())
def _read_data(reader, idx):
with reader:
data_store[idx] = dataserializer.deserialize(
reader.read())
