def test_keys_to_values(sc):
x = arange(4 * 7 * 9 * 6).reshape(4, 7, 9, 6)
b = array(x, sc, (0, 1))
c = b.chunk((4, 2))
assert allclose(
x,
c.keys_to_values((0, )).unchunk().toarray().transpose(1, 0, 2, 3))
assert allclose(x, c.keys_to_values((1, )).unchunk().toarray())
assert allclose(x, c.keys_to_values((1, ), size=(3, )).unchunk().toarray())
assert allclose(x, c.keys_to_values((0, 1)).unchunk().toarray())
assert allclose(x,
c.keys_to_values((0, 1), size=(2, 3)).unchunk().toarray())
assert allclose(x, c.keys_to_values(()).unchunk().toarray())
b = array(x, sc, range(4))
c = b.chunk(())
assert allclose(x, c.keys_to_values((3, )).unchunk().toarray())
assert allclose(
x,
c.keys_to_values((0, 1)).unchunk().toarray().transpose(2, 3, 0, 1))
b = array(x, sc, (0, ))
c = b.chunk((2, 3, 4))
assert allclose(x, c.keys_to_values((0, )).unchunk().toarray())
def test_getitem_slice(sc):
x = arange(6 * 6).reshape((6, 6))
b = array(x, sc, axis=0)
assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
assert allclose(b[:2, :2].toarray(), x[:2, :2])
assert allclose(b[1:, 1:].toarray(), x[1:, 1:])
assert allclose(b[5:1:-1, 5:1:-1].toarray(), x[5:1:-1, 5:1:-1])
assert allclose(b[10:-10:-2, 10:-10:-2].toarray(), x[10:-10:-2, 10:-10:-2])
assert allclose(b[-5:-1, -5:-1].toarray(), x[-5:-1, -5:-1])
assert allclose(b[-1:-5:-2, -1:-5:-2].toarray(), x[-1:-5:-2, -1:-5:-2])
b = array(x, sc, axis=(0, 1))
assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
assert allclose(b[:2, :2].toarray(), x[:2, :2])
assert allclose(b[1:, 1:].toarray(), x[1:, 1:])
assert allclose(b[5:1:-1, 5:1:-1].toarray(), x[5:1:-1, 5:1:-1])
assert allclose(b[10:-10:-2, 10:-10:-2].toarray(), x[10:-10:-2, 10:-10:-2])
assert allclose(b[-5:-1, -5:-1].toarray(), x[-5:-1, -5:-1])
assert allclose(b[-1:-5:-2, -1:-5:-2].toarray(), x[-1:-5:-2, -1:-5:-2])
def test_concatenate(sc):
from numpy import concatenate as npconcatenate
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc, axis=0)
bb = concatenate((b, b), axis=0)
assert allclose(npconcatenate((x, x), axis=0), bb.toarray())
bb = concatenate((b, b), axis=1)
assert allclose(npconcatenate((x, x), axis=1), bb.toarray())
bb = concatenate((b, b), axis=2)
assert allclose(npconcatenate((x, x), axis=2), bb.toarray())
b = array(x, sc, axis=(0, 1))
bb = concatenate((b, b), axis=0)
assert allclose(npconcatenate((x, x), axis=0), bb.toarray())
b = array(x, sc, axis=(0, 1))
bb = concatenate((b, b), axis=1)
assert allclose(npconcatenate((x, x), axis=1), bb.toarray())
b = array(x, sc, axis=(0, 1))
bb = concatenate((b, b), axis=2)
assert allclose(npconcatenate((x, x), axis=2), bb.toarray())
def test_getitem_int(sc):
x = arange(2*3).reshape((2, 3))
b = array(x, sc, axis=0)
assert allclose(b[0, 0], x[0, 0])
assert allclose(b[0, 1], x[0, 1])
assert allclose(b[0, 0:1], x[0, 0:1])
assert allclose(b[1, 2], x[1, 2])
assert allclose(b[0], x[0])
assert allclose(b[[0]], x[[0]])
assert allclose(b[(0)], x[(0)])
assert allclose(b[[1], [2]], x[[1], [2]])
assert allclose(b[[1], 2], x[[1], 2])
assert allclose(b[-1, -2], x[-1, -2])
b = array(x, sc, axis=(0, 1))
assert allclose(b[0, 0], x[0, 0])
assert allclose(b[0, 1], x[0, 1])
assert allclose(b[0, 0:1], x[0, 0:1])
assert allclose(b[1, 2], x[1, 2])
assert allclose(b[0], x[0])
assert allclose(b[[0]], x[[0]])
assert allclose(b[(0)], x[(0)])
assert allclose(b[[1], [2]], x[[1], [2]])
assert allclose(b[[1], 2], x[[1], 2])
assert allclose(b[-1, -2], x[-1, -2])
def test_getitem_int(sc):
x = arange(2 * 3).reshape((2, 3))
b = array(x, sc, axis=0)
assert allclose(b[0, 0], x[0, 0])
assert allclose(b[0, 1], x[0, 1])
assert allclose(b[0, 0:1], x[0, 0:1])
assert allclose(b[1, 2], x[1, 2])
assert allclose(b[0], x[0])
assert allclose(b[[0]], x[[0]])
assert allclose(b[(0)], x[(0)])
assert allclose(b[[1], [2]], x[[1], [2]])
assert allclose(b[[1], 2], x[[1], 2])
assert allclose(b[-1, -2], x[-1, -2])
b = array(x, sc, axis=(0, 1))
assert allclose(b[0, 0], x[0, 0])
assert allclose(b[0, 1], x[0, 1])
assert allclose(b[0, 0:1], x[0, 0:1])
assert allclose(b[1, 2], x[1, 2])
assert allclose(b[0], x[0])
assert allclose(b[[0]], x[[0]])
assert allclose(b[(0)], x[(0)])
assert allclose(b[[1], [2]], x[[1], [2]])
assert allclose(b[[1], 2], x[[1], 2])
assert allclose(b[-1, -2], x[-1, -2])
def test_getitem_slice(sc):
x = arange(6*6).reshape((6, 6))
b = array(x, sc, axis=0)
assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
assert allclose(b[:2, :2].toarray(), x[:2, :2])
assert allclose(b[1:, 1:].toarray(), x[1:, 1:])
assert allclose(b[5:1:-1, 5:1:-1].toarray(), x[5:1:-1, 5:1:-1])
assert allclose(b[10:-10:-2, 10:-10:-2].toarray(), x[10:-10:-2, 10:-10:-2])
assert allclose(b[-5:-1, -5:-1].toarray(), x[-5:-1, -5:-1])
assert allclose(b[-1:-5:-2, -1:-5:-2].toarray(), x[-1:-5:-2, -1:-5:-2])
b = array(x, sc, axis=(0, 1))
assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
assert allclose(b[:2, :2].toarray(), x[:2, :2])
assert allclose(b[1:, 1:].toarray(), x[1:, 1:])
assert allclose(b[5:1:-1, 5:1:-1].toarray(), x[5:1:-1, 5:1:-1])
assert allclose(b[10:-10:-2, 10:-10:-2].toarray(), x[10:-10:-2, 10:-10:-2])
assert allclose(b[-5:-1, -5:-1].toarray(), x[-5:-1, -5:-1])
assert allclose(b[-1:-5:-2, -1:-5:-2].toarray(), x[-1:-5:-2, -1:-5:-2])
def test_unchunk(sc):
x = arange(4 * 6).reshape(1, 4, 6)
b = array(x, sc)
assert allclose(b.chunk((2, 3)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((3, 4)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((4, 6)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk('0.1').unchunk().toarray(), b.toarray())
assert allclose(b.chunk().unchunk().toarray(), b.toarray())
x = arange(4 * 5 * 10).reshape(1, 4, 5, 10)
b = array(x, sc)
assert allclose(b.chunk((4, 5, 10)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((1, 1, 1)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((3, 3, 3)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((3, 3, 3)).unchunk().toarray(), b.toarray())
x = arange(4 * 6).reshape(4, 6)
b = array(x, sc, (0, 1))
assert allclose(b.chunk(()).unchunk().toarray(), b.toarray())
b = array(x, sc, (0, ))
assert allclose(b.chunk((2)).unchunk().toarray(), b.toarray())
def test_unchunk(sc):
x = arange(4*6).reshape(1, 4, 6)
b = array(x, sc)
assert allclose(b.chunk((2, 3)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((3, 4)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((4, 6)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk('0.1').unchunk().toarray(), b.toarray())
assert allclose(b.chunk().unchunk().toarray(), b.toarray())
x = arange(4*5*10).reshape(1, 4, 5, 10)
b = array(x, sc)
assert allclose(b.chunk((4, 5, 10)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((1, 1, 1)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((3, 3, 3)).unchunk().toarray(), b.toarray())
assert allclose(b.chunk((3, 3, 3)).unchunk().toarray(), b.toarray())
x = arange(4*6).reshape(4, 6)
b = array(x, sc, (0, 1))
assert allclose(b.chunk(()).unchunk().toarray(), b.toarray())
b = array(x, sc, (0,))
assert allclose(b.chunk((2)).unchunk().toarray(), b.toarray())
def test_split(sc):
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc, axis=0)
assert b.split == 1
b = array(x, sc, axis=(0, 1))
assert b.split == 2
def test_value_shape(sc):
x = arange(2 * 3).reshape((2, 3))
b = array(x, sc)
assert b.values.shape == (3, )
x = arange(2 * 3 * 4).reshape((2, 3, 4))
b = array(x, sc, axis=0)
assert b.values.shape == (3, 4)
def test_shape(sc):
x = arange(2*3).reshape((2, 3))
b = array(x, sc)
assert b.shape == x.shape
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc)
assert b.shape == x.shape
def test_array_errors(sc):
x = arange(2*3*4).reshape((2, 3, 4))
with pytest.raises(ValueError):
array(x, sc, axis=-1)
with pytest.raises(ValueError):
array(x, sc, axis=(0, 1, 2, 3))
def test_concatenate(sc):
from numpy import concatenate
x = arange(2*3).reshape((2, 3))
b = array(x, sc)
c = array(x)
assert allclose(b.concatenate(x).toarray(), concatenate((x, x)))
assert allclose(b.concatenate(b).toarray(), concatenate((x, x)))
assert allclose(b.concatenate(c).toarray(), concatenate((x, x)))
def test_t(sc):
a = arange(2 * 3 * 4 * 5).reshape((2, 3, 4, 5))
b = array(a, sc, axis=0)
assert allclose(b.T.toarray(), b.toarray().T)
b = array(a, sc, axis=(0, 1))
assert allclose(b.T.toarray(), b.toarray().T)
def test_key_shape(sc):
x = arange(2*3).reshape((2, 3))
b = array(x, sc)
assert b.keys.shape == (2,)
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc, axis=(0, 1))
assert b.keys.shape == (2, 3)
def test_key_shape(sc):
x = arange(2 * 3).reshape((2, 3))
b = array(x, sc)
assert b.keys.shape == (2, )
x = arange(2 * 3 * 4).reshape((2, 3, 4))
b = array(x, sc, axis=(0, 1))
assert b.keys.shape == (2, 3)
def test_t(sc):
a = arange(2*3*4*5).reshape((2, 3, 4, 5))
b = array(a, sc, axis=0)
assert allclose(b.T.toarray(), b.toarray().T)
b = array(a, sc, axis=(0, 1))
assert allclose(b.T.toarray(), b.toarray().T)
def test_value_shape(sc):
x = arange(2*3).reshape((2, 3))
b = array(x, sc)
assert b.values.shape == (3,)
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc, axis=0)
assert b.values.shape == (3, 4)
def test_mask(sc):
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc, axis=0)
assert b.mask == (1, 0, 0)
b = array(x, sc, axis=(0, 1))
assert b.mask == (1, 1, 0)
b = array(x, sc, axis=(0, 1, 2))
assert b.mask == (1, 1, 1)
def test_getitem_list_array(sc):
x = arange(3 * 3 * 4).reshape((3, 3, 4))
rows = [[0, 0], [1, 1]]
cols = [[0, 2], [0, 2]]
dept = [[0, 3], [0, 3]]
b = array(x, sc, axis=0)
assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])
b = array(x, sc, axis=(0, 1))
assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])
def test_getitem_list_array(sc):
x = arange(3*3*4).reshape((3, 3, 4))
rows = [[0, 0], [1, 1]]
cols = [[0, 2], [0, 2]]
dept = [[0, 3], [0, 3]]
b = array(x, sc, axis=0)
assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])
b = array(x, sc, axis=(0, 1))
assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])
def test_getitem_list(sc):
x = arange(3*3*4).reshape((3, 3, 4))
b = array(x, sc, axis=0)
assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])
b = array(x, sc, axis=(0,1))
assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])
def test_array(sc):
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc)
assert isinstance(b, BoltArraySpark)
assert allclose(x, b.toarray())
b = array(x, sc, axis=0)
assert isinstance(b, BoltArraySpark)
assert allclose(x, b.toarray())
b = array(x, sc, axis=(0, 1))
assert isinstance(b, BoltArraySpark)
assert allclose(x, b.toarray())
def test_filter(sc):
x = arange(2*3*4).reshape(2, 3, 4)
b = array(x, sc, axis=0)
# Test all filter functionality when the base array is split after the first axis
generic.filter_suite(x, b)
# Split the BoltArraySpark after the second axis and rerun the tests
b = array(x, sc, axis=(0, 1))
generic.filter_suite(x, b)
# Split the BoltArraySpark after the third axis (scalar values) and rerun the tests
b = array(x, sc, axis=(0, 1, 2))
generic.filter_suite(x, b)
def test_transpose_keys(sc):
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc, axis=(0, 1))
c = b.keys.transpose((1, 0))
assert c.keys.shape == (3, 2)
assert allclose(c.toarray(), x.transpose((1, 0, 2)))
b = array(x, sc, axis=0)
c = b.keys.transpose((0,))
assert allclose(c.toarray(), x)
b = array(x, sc, axis=(0, 1))
c = b.keys.transpose((0, 1))
assert allclose(c.toarray(), x)
def test_astype(sc):
from numpy import ones as npones
a = npones(2**8, dtype=int64)
b = array(a, sc, dtype=int64)
c = b.astype(bool)
assert c.dtype == dtype(bool)
dtypes = c._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
b = ones((100, 100), sc, dtype=int64)
c = b.astype(bool)
assert c.dtype == dtype(bool)
dtypes = c._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
b = ones((100, 100), sc)
c = b.astype(bool)
assert c.dtype == dtype(bool)
dtypes = c._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
def test_map_errors(sc):
x = arange(4 * 8 * 8).reshape(4, 8, 8)
b = array(x, sc)
c = b.chunk(size=(4, 8))
# changing the size of a chunked axis
def f(x):
return x[:2, :]
with pytest.raises(ValueError):
c.map(f)
with pytest.raises(ValueError):
c.map(f, value_shape=(2, 8))
# dropping dimensions
def f(x):
return x[0, :]
with pytest.raises(NotImplementedError):
c.map(f)
with pytest.raises(NotImplementedError):
c.map(f, value_shape=(4, ))
def test_reshape_values_errors(sc):
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc, axis=(0, 1))
with pytest.raises(ValueError):
b.values.reshape((2, 3, 4))
def test_map_errors(sc):
x = arange(4*8*8).reshape(4, 8, 8)
b = array(x, sc)
c = b.chunk(size=(4, 8))
# changing the size of a chunked axis
def f(x):
return x[:2, :]
with pytest.raises(ValueError):
c.map(f)
with pytest.raises(ValueError):
c.map(f, value_shape=(2, 8))
# dropping dimensions
def f(x):
return x[0, :]
with pytest.raises(NotImplementedError):
c.map(f)
with pytest.raises(NotImplementedError):
c.map(f, value_shape=(4,))
def test_properties(sc):
x = arange(4*6).reshape(1, 4, 6)
b = array(x, sc)
assert b.chunk(size=(2, 3)).uniform is True
assert b.chunk(size=(2, 4)).uniform is False
def test_transpose_values(sc):
x = arange(2*3*4).reshape((2, 3, 4))
b = array(x, sc, axis=0)
c = b.values.transpose((1, 0))
assert c.values.shape == (4, 3)
assert allclose(c.toarray(), x.transpose((0, 2, 1)))
b = array(x, sc, axis=0)
c = b.values.transpose((0, 1))
assert allclose(c.toarray(), x)
b = array(x, sc, axis=(0, 1))
c = b.values.transpose((0,))
assert allclose(c.toarray(), x.reshape((2, 3, 4)))
def test_args(sc):
x = arange(4*6).reshape(1, 4, 6)
b = array(x, sc)
with pytest.raises(ValueError):
b.chunk(size=(5, 6))
def test_properties(sc):
x = arange(4 * 6).reshape(1, 4, 6)
b = array(x, sc)
assert b.chunk(size=(2, 3)).uniform is True
assert b.chunk(size=(2, 4)).uniform is False
def test_filter():
x = arange(2*3*4).reshape(2, 3, 4)
b = array(x)
# Test all generic filter functionality
generic.filter_suite(x, b)
def test_args(sc):
x = arange(4 * 6).reshape(1, 4, 6)
b = array(x, sc)
with pytest.raises(ValueError):
b.chunk(size=(5, 6))
def test_repartition(sc):
x = arange(2 * 3).reshape((2, 3))
b = array(x, sc)
assert b._ordered
b = b.repartition(10)
assert not b._ordered
assert b._rdd.getNumPartitions() == 10
def test_values_to_keys(sc):
x = arange(4*7*9*6).reshape(4, 7, 9, 6)
b = array(x, sc, (0, 1))
c = b.chunk((4, 2))
assert allclose(x, c.values_to_keys((0,)).unchunk().toarray())
assert allclose(x, c.values_to_keys((1,)).unchunk().toarray().transpose(0, 1, 3, 2))
assert allclose(x, c.values_to_keys((0, 1)).unchunk().toarray())
assert allclose(x, c.values_to_keys(()).unchunk().toarray())
b = array(x, sc, (0,))
c = b.chunk((2, 3, 4))
assert allclose(x, c.values_to_keys((0,)).unchunk().toarray())
assert allclose(x, c.values_to_keys((0, 1)).unchunk().toarray())
def test_values_to_keys(sc):
x = arange(4*7*9*6).reshape(4, 7, 9, 6)
b = array(x, sc, (0, 1))
c = b.chunk((4, 2))
assert allclose(x, c.values_to_keys((0,)).unchunk().toarray())
assert allclose(x, c.values_to_keys((1,)).unchunk().toarray().transpose(0, 1, 3, 2))
assert allclose(x, c.values_to_keys((0, 1)).unchunk().toarray())
assert allclose(x, c.values_to_keys(()).unchunk().toarray())
b = array(x, sc, (0,))
c = b.chunk((2, 3, 4))
assert allclose(x, c.values_to_keys((0,)).unchunk().toarray())
assert allclose(x, c.values_to_keys((0, 1)).unchunk().toarray())
def test_transpose_keys(sc):
x = arange(2 * 3 * 4).reshape((2, 3, 4))
b = array(x, sc, axis=(0, 1))
c = b.keys.transpose((1, 0))
assert c.keys.shape == (3, 2)
assert allclose(c.toarray(), x.transpose((1, 0, 2)))
b = array(x, sc, axis=0)
c = b.keys.transpose((0, ))
assert allclose(c.toarray(), x)
b = array(x, sc, axis=(0, 1))
c = b.keys.transpose((0, 1))
assert allclose(c.toarray(), x)
def test_transpose_values(sc):
x = arange(2 * 3 * 4).reshape((2, 3, 4))
b = array(x, sc, axis=0)
c = b.values.transpose((1, 0))
assert c.values.shape == (4, 3)
assert allclose(c.toarray(), x.transpose((0, 2, 1)))
b = array(x, sc, axis=0)
c = b.values.transpose((0, 1))
assert allclose(c.toarray(), x)
b = array(x, sc, axis=(0, 1))
c = b.values.transpose((0, ))
assert allclose(c.toarray(), x.reshape((2, 3, 4)))
def test_reshape_values_errors(sc):
x = arange(2 * 3 * 4).reshape((2, 3, 4))
b = array(x, sc, axis=(0, 1))
with pytest.raises(ValueError):
b.values.reshape((2, 3, 4))
def test_reshape(sc):
old_shape = (6, 10, 4, 12)
a = arange(prod(old_shape)).reshape(old_shape)
b = array(a, sc, axis=(0, 1))
# keys only
new_shape = (15, 4, 4, 12)
assert allclose(
b.reshape(new_shape).toarray(),
b.toarray().reshape(new_shape))
# values only
new_shape = (6, 10, 24, 2)
assert allclose(
b.reshape(new_shape).toarray(),
b.toarray().reshape(new_shape))
# keys and values, independent
new_shape = (15, 4, 24, 2)
assert allclose(
b.reshape(new_shape).toarray(),
b.toarray().reshape(new_shape))
# keys and values, mixing
new_shape = (6, 4, 10, 12)
with pytest.raises(NotImplementedError):
b.reshape(new_shape)
def test_dtype(sc):
a = arange(2 ** 8, dtype=int64)
b = array(a, sc, dtype=int64)
assert a.dtype == b.dtype
assert b.dtype == dtype(int64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(int64)
a = arange(2.0 ** 8)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(float64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(float64)
a = arange(2 ** 8)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(int64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(int64)
from numpy import ones as npones
a = npones(2 ** 8, dtype=bool)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(bool)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
b = ones(2 ** 8, sc)
assert b.dtype == dtype(float64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(float64)
b = ones(2 ** 8, sc, dtype=bool)
assert b.dtype == dtype(bool)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
def test_swap(sc):
a = arange(2 ** 8).reshape(*(8 * [2]))
b = array(a, sc, axis=(0, 1, 2, 3))
bs = b.swap((1, 2), (0, 3), size=(2, 2))
at = a.transpose((0, 3, 4, 7, 1, 2, 5, 6))
assert allclose(at, bs.toarray())
bs = b.swap((1, 2), (0, 3), size="50")
at = a.transpose((0, 3, 4, 7, 1, 2, 5, 6))
assert allclose(at, bs.toarray())
bs = b.swap((1, 2), (0, 3))
at = a.transpose((0, 3, 4, 7, 1, 2, 5, 6))
assert allclose(at, bs.toarray())
bs = b.swap((), (0, 1, 2, 3))
at = a
assert allclose(at, bs.toarray())
bs = b.swap(0, 0)
at = a.transpose((1, 2, 3, 4, 0, 5, 6, 7))
assert allclose(at, bs.toarray())
bs = b.swap([], 0)
at = a.transpose((0, 1, 2, 3, 4, 5, 6, 7))
assert allclose(at, bs.toarray())
assert bs.split == 5
bs = b.swap(0, [])
at = a.transpose((1, 2, 3, 0, 4, 5, 6, 7))
assert allclose(at, bs.toarray())
assert bs.split == 3
b = array(a, sc, axis=range(8))
bs = b.swap([0, 1], [])
at = a.transpose((2, 3, 4, 5, 6, 7, 0, 1))
assert allclose(at, bs.toarray())
assert bs.split == 6
a = arange(2 * 3 * 4).reshape(2, 3, 4)
b = array(a, sc, axis=(0,))
bs = b.swap((0,), (0, 1))
at = a.transpose(1, 2, 0)
assert allclose(at, bs.toarray())
def test_swap(sc):
a = arange(2**8).reshape(*(8 * [2]))
b = array(a, sc, axis=(0, 1, 2, 3))
bs = b.swap((1, 2), (0, 3), size=(2, 2))
at = a.transpose((0, 3, 4, 7, 1, 2, 5, 6))
assert allclose(at, bs.toarray())
bs = b.swap((1, 2), (0, 3), size="50")
at = a.transpose((0, 3, 4, 7, 1, 2, 5, 6))
assert allclose(at, bs.toarray())
bs = b.swap((1, 2), (0, 3))
at = a.transpose((0, 3, 4, 7, 1, 2, 5, 6))
assert allclose(at, bs.toarray())
bs = b.swap((), (0, 1, 2, 3))
at = a
assert allclose(at, bs.toarray())
bs = b.swap(0, 0)
at = a.transpose((1, 2, 3, 4, 0, 5, 6, 7))
assert allclose(at, bs.toarray())
bs = b.swap([], 0)
at = a.transpose((0, 1, 2, 3, 4, 5, 6, 7))
assert allclose(at, bs.toarray())
assert bs.split == 5
bs = b.swap(0, [])
at = a.transpose((1, 2, 3, 0, 4, 5, 6, 7))
assert allclose(at, bs.toarray())
assert bs.split == 3
b = array(a, sc, axis=range(8))
bs = b.swap([0, 1], [])
at = a.transpose((2, 3, 4, 5, 6, 7, 0, 1))
assert allclose(at, bs.toarray())
assert bs.split == 6
a = arange(2 * 3 * 4).reshape(2, 3, 4)
b = array(a, sc, axis=(0, ))
bs = b.swap((0, ), (0, 1))
at = a.transpose(1, 2, 0)
assert allclose(at, bs.toarray())
def fromarray(values, labels=None, npartitions=None, engine=None):
"""
Load images from an array.
First dimension will be used to index images,
so remaining dimensions after the first should
be the dimensions of the images,
e.g. (3, 100, 200) for 3 x (100, 200) images
Parameters
----------
values : array-like
The array of images. Can be a numpy array,
a bolt array, or an array-like.
labels : array, optional, default = None
Labels for records. If provided, should be one-dimensional.
npartitions : int, default = None
Number of partitions for parallelization (spark only)
engine : object, default = None
Computational engine (e.g. a SparkContext for spark)
"""
from .images import Images
import bolt
if isinstance(values, bolt.spark.array.BoltArraySpark):
return Images(values)
values = asarray(values)
if values.ndim < 2:
raise ValueError('Array for images must have at least 2 dimensions, got %g' % values.ndim)
if values.ndim == 2:
values = expand_dims(values, 0)
shape = None
dtype = None
for im in values:
if shape is None:
shape = im.shape
dtype = im.dtype
if not im.shape == shape:
raise ValueError('Arrays must all be of same shape; got both %s and %s' %
(str(shape), str(im.shape)))
if not im.dtype == dtype:
raise ValueError('Arrays must all be of same data type; got both %s and %s' %
(str(dtype), str(im.dtype)))
if spark and isinstance(engine, spark):
if not npartitions:
npartitions = engine.defaultParallelism
values = bolt.array(values, context=engine, npartitions=npartitions, axis=(0,))
values._ordered = True
return Images(values)
return Images(values, labels=labels)
def test_swapaxes(sc):
a = arange(2 * 3 * 4 * 5).reshape((2, 3, 4, 5))
b = array(a, sc, axis=(0, 1))
assert allclose(b.swapaxes(1, 2).toarray(), b.toarray().swapaxes(1, 2))
assert allclose(b.swapaxes(0, 1).toarray(), b.toarray().swapaxes(0, 1))
assert allclose(b.swapaxes(2, 3).toarray(), b.toarray().swapaxes(2, 3))
def test_cache(sc):
x = arange(2*3).reshape((2, 3))
b = array(x, sc)
b.cache()
assert b._rdd.is_cached
b.unpersist()
assert not b._rdd.is_cached
def test_dtype(sc):
a = arange(2**8, dtype=int64)
b = array(a, sc, dtype=int64)
assert a.dtype == b.dtype
assert b.dtype == dtype(int64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(int64)
a = arange(2.0**8)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(float64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(float64)
a = arange(2**8)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(int64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(int64)
from numpy import ones as npones
a = npones(2**8, dtype=bool)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(bool)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
b = ones(2**8, sc)
assert b.dtype == dtype(float64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(float64)
b = ones(2**8, sc, dtype=bool)
assert b.dtype == dtype(bool)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)