def test_to_binary(tmpdir, eng):
a = arange(8, dtype='int16').reshape((4, 2))
p = str(tmpdir) + '/data'
fromarray(a, npartitions=1, engine=eng).tobinary(p)
files = [os.path.basename(f) for f in glob.glob(str(tmpdir) + '/data/*')]
assert sorted(files) == ['SUCCESS', 'conf.json', 'series-00000.bin']
with open(str(tmpdir) + '/data/conf.json', 'r') as f:
conf = json.load(f)
assert conf['shape'] == [4, 2]
assert conf['dtype'] == 'int16'
def test_nanmin(eng):
arr = array([arange(8), arange(8)]).astype(float64)
data = fromarray(arr, engine=eng)
val = data.nanmin().toarray()
expected = nanmin(data.toarray(), axis=0)
assert allclose(val, expected)
assert str(val.dtype) == 'float64'
arr[0, 4] = nan
arr[1, 3] = nan
arr[1, 4] = nan
data = fromarray(arr, engine=eng)
val = data.nanmin().toarray()
expected = nanmin(data.toarray(), axis=0)
assert allclose(val, expected, equal_nan=True)
assert str(val.dtype) == 'float64'
def test_nanmin(eng):
arr = array([arange(8), arange(8)]).astype(float64)
data = fromarray(arr, engine=eng)
val = data.nanmin().toarray()
expected = nanmin(data.toarray(), axis=0)
assert allclose(val, expected)
assert str(val.dtype) == 'float64'
arr[0, 4] = nan
arr[1, 3] = nan
arr[1, 4] = nan
data = fromarray(arr, engine=eng)
val = data.nanmin().toarray()
expected = nanmin(data.toarray(), axis=0)
assert allclose(val, expected, equal_nan=True)
assert str(val.dtype) == 'float64'
def test_from_array_vector(eng):
a = arange(8, dtype='int16').reshape((4, 2))
data = fromarray(a, engine=eng)
assert data.shape == (4, 2)
assert data.dtype == 'int16'
assert allclose(data.index, [0, 1])
assert allclose(data.toarray(), a)
def test_to_binary_roundtrip_3d(tmpdir, eng):
a = arange(16, dtype='int16').reshape((4, 2, 2))
p = str(tmpdir) + '/data'
data = fromarray(a, npartitions=1, engine=eng)
data.tobinary(p)
loaded = frombinary(p, engine=eng)
assert allclose(data.toarray(), loaded.toarray())
def tolocal(self):
"""
Convert to local mode.
"""
from thunder.series.readers import fromarray
if self.mode == 'local':
logging.getLogger('thunder').warn('images already in local mode')
pass
return fromarray(self.toarray(), index=self.index, labels=self.labels)
def tolocal(self):
"""
Convert to local mode.
"""
from thunder.series.readers import fromarray
if self.mode == 'local':
logging.getLogger('thunder').warn('images already in local mode')
pass
return fromarray(self.toarray(), index=self.index, labels=self.labels)
def test_from_array_bolt(eng):
a = arange(8, dtype='int16').reshape((4, 2))
if eng is not None:
b = barray(a, context=eng)
else:
b = barray(a)
data = fromarray(b, engine=eng)
assert data.shape == (4, 2)
assert data.dtype == 'int16'
assert allclose(data.index, [0, 1])
assert allclose(data.toarray(), a)
def tospark(self, engine=None):
"""
Convert to spark mode.
"""
from thunder.series.readers import fromarray
if self.mode == 'spark':
logging.getLogger('thunder').warn('images already in local mode')
pass
if engine is None:
raise ValueError('Must provide SparkContext')
return fromarray(self.toarray(), index=self.index, labels=self.labels, engine=engine)
def tospark(self, engine=None):
"""
Convert to spark mode.
"""
from thunder.series.readers import fromarray
if self.mode == 'spark':
logging.getLogger('thunder').warn('images already in local mode')
pass
if engine is None:
raise ValueError('Must provide SparkContext')
return fromarray(self.toarray(), index=self.index, labels=self.labels, engine=engine)
def test_reshape(eng):
original = fromarray(arange(72).reshape(6, 6, 2), engine=eng)
arr = original.toarray()
assert allclose(arr.reshape(12, 3, 2), original.reshape(12, 3, 2).toarray())
assert allclose(arr.reshape(36, 2), original.reshape(36, 2).toarray())
assert allclose(arr.reshape(4, 3, 3, 2), original.reshape(4, 3, 3, 2).toarray())
# must conserve number of elements
with pytest.raises(ValueError):
original.reshape(6, 3, 2)
# cannot change length of series
with pytest.raises(ValueError):
original.reshape(6, 3, 4)
def test_reshape(eng):
original = fromarray(arange(72).reshape(6, 6, 2), engine=eng)
arr = original.toarray()
assert allclose(arr.reshape(12, 3, 2), original.reshape(12, 3, 2).toarray())
assert allclose(arr.reshape(36, 2), original.reshape(36, 2).toarray())
assert allclose(arr.reshape(4, 3, 3, 2), original.reshape(4, 3, 3, 2).toarray())
# must conserve number of elements
with pytest.raises(ValueError):
original.reshape(6, 3, 2)
# cannot change length of series
with pytest.raises(ValueError):
original.reshape(6, 3, 4)
def test_from_array_index(eng):
a = arange(8, dtype='int16').reshape((4, 2))
data = fromarray(a, index=[2, 3], engine=eng)
assert allclose(data.index, [2, 3])
def test_flatten(eng):
arr = arange(2 * 2 * 5).reshape(2, 2, 5)
data = fromarray(arr, engine=eng)
assert data.flatten().shape == (4, 5)
assert allclose(data.flatten().toarray(), arr.reshape(2 * 2, 5))
def test_flatten(eng):
arr = arange(2*2*5).reshape(2, 2, 5)
data = fromarray(arr, engine=eng)
assert data.flatten().shape == (4, 5)
assert allclose(data.flatten().toarray(), arr.reshape(2*2, 5))
