def test_astype(sc):
from numpy import ones as npones
a = npones(2**8, dtype=int64)
b = array(a, sc, dtype=int64)
c = b.astype(bool)
assert c.dtype == dtype(bool)
dtypes = c._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
b = ones((100, 100), sc, dtype=int64)
c = b.astype(bool)
assert c.dtype == dtype(bool)
dtypes = c._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
b = ones((100, 100), sc)
c = b.astype(bool)
assert c.dtype == dtype(bool)
dtypes = c._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
def test_squeeze(sc):
from numpy import ones as npones
x = npones((1, 2, 1, 4))
b = ones((1, 2, 1, 4), sc, axis=0)
assert allclose(b.squeeze().toarray(), x.squeeze())
assert allclose(b.squeeze((0, 2)).toarray(), x.squeeze((0, 2)))
assert allclose(b.squeeze(0).toarray(), x.squeeze(0))
assert allclose(b.squeeze(2).toarray(), x.squeeze(2))
assert b.squeeze().split == 0
assert b.squeeze((0, 2)).split == 0
assert b.squeeze(2).split == 1
x = npones((1, 2, 1, 4))
b = ones((1, 2, 1, 4), sc, axis=(0, 1))
assert allclose(b.squeeze().toarray(), x.squeeze())
assert allclose(b.squeeze((0, 2)).toarray(), x.squeeze((0, 2)))
assert allclose(b.squeeze(0).toarray(), x.squeeze(0))
assert allclose(b.squeeze(2).toarray(), x.squeeze(2))
assert b.squeeze().split == 1
assert b.squeeze((0, 2)).split == 1
assert b.squeeze(2).split == 2
x = npones((1, 1, 1, 1))
b = ones((1, 1, 1, 1), sc, axis=(0, 1))
assert allclose(b.squeeze().toarray(), x.squeeze())
def test_squeeze(sc):
from numpy import ones as npones
x = npones((1, 2, 1, 4))
b = ones((1, 2, 1, 4), sc, axis=0)
assert allclose(b.squeeze().toarray(), x.squeeze())
assert allclose(b.squeeze((0, 2)).toarray(), x.squeeze((0, 2)))
assert allclose(b.squeeze(0).toarray(), x.squeeze(0))
assert allclose(b.squeeze(2).toarray(), x.squeeze(2))
assert b.squeeze().split == 0
assert b.squeeze((0, 2)).split == 0
assert b.squeeze(2).split == 1
x = npones((1, 2, 1, 4))
b = ones((1, 2, 1, 4), sc, axis=(0, 1))
assert allclose(b.squeeze().toarray(), x.squeeze())
assert allclose(b.squeeze((0, 2)).toarray(), x.squeeze((0, 2)))
assert allclose(b.squeeze(0).toarray(), x.squeeze(0))
assert allclose(b.squeeze(2).toarray(), x.squeeze(2))
assert b.squeeze().split == 1
assert b.squeeze((0, 2)).split == 1
assert b.squeeze(2).split == 2
x = npones((1, 1, 1, 1))
b = ones((1, 1, 1, 1), sc, axis=(0, 1))
assert allclose(b.squeeze().toarray(), x.squeeze())
def test_ones(sc):
from numpy import ones as npones
x = npones((2, 3, 4))
b = ones((2, 3, 4), sc)
assert allclose(x, b.toarray())
x = npones(5)
b = ones(5, sc)
assert allclose(x, b.toarray())
def test_map_drop_dim(sc):
a = ones((2, 20, 10, 3), sc)
c = a.chunk((10, 5, 3))
assert c.map(lambda x: x[:, :, 0:2]).unchunk().toarray().shape == (2, 20, 10, 2)
assert c.map(lambda x: x[:, :, 0]).unchunk().toarray().shape == (2, 20, 10, 1)
assert c.map(lambda x: x[:, :, [0]]).unchunk().toarray().shape == (2, 20, 10, 1)
a = ones((2, 20, 10), sc)
c = a.chunk((10, 5))
assert c.map(lambda x: x[:, 0:2]).unchunk().toarray().shape == (2, 20, 4)
assert c.map(lambda x: x[:, 0]).unchunk().toarray().shape == (2, 20, 2)
assert c.map(lambda x: x[:, [0]]).unchunk().toarray().shape == (2, 20, 2)
def test_dtype(sc):
a = arange(2 ** 8, dtype=int64)
b = array(a, sc, dtype=int64)
assert a.dtype == b.dtype
assert b.dtype == dtype(int64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(int64)
a = arange(2.0 ** 8)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(float64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(float64)
a = arange(2 ** 8)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(int64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(int64)
from numpy import ones as npones
a = npones(2 ** 8, dtype=bool)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(bool)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
b = ones(2 ** 8, sc)
assert b.dtype == dtype(float64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(float64)
b = ones(2 ** 8, sc, dtype=bool)
assert b.dtype == dtype(bool)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
def test_dtype(sc):
a = arange(2**8, dtype=int64)
b = array(a, sc, dtype=int64)
assert a.dtype == b.dtype
assert b.dtype == dtype(int64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(int64)
a = arange(2.0**8)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(float64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(float64)
a = arange(2**8)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(int64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(int64)
from numpy import ones as npones
a = npones(2**8, dtype=bool)
b = array(a, sc)
assert a.dtype == b.dtype
assert b.dtype == dtype(bool)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
b = ones(2**8, sc)
assert b.dtype == dtype(float64)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(float64)
b = ones(2**8, sc, dtype=bool)
assert b.dtype == dtype(bool)
dtypes = b._rdd.map(lambda x: x[1].dtype).collect()
for dt in dtypes:
assert dt == dtype(bool)
def test_stacked_shape_inference(sc):
from numpy import ones as npones
a = ones((100, 2), sc)
a._rdd = a._rdd.partitionBy(2)
s = a.stack(5)
n = s.tordd().count()
# operations that preserve keys
assert s.map(lambda x: x * 2).unstack().shape == (100, 2)
assert s.map(lambda x: x.sum(axis=1)).unstack().shape == (100, )
assert s.map(lambda x: tile(x, (1, 2))).unstack().shape == (100, 4)
# operations that create new keys
assert s.map(lambda x: npones((2, 2))).unstack().shape == (n, 2, 2)
assert s.map(lambda x: x.sum(axis=0)).unstack().shape == (n, 2)
assert s.map(lambda x: asarray([2])).unstack().toarray().shape == (n, 1)
assert s.map(lambda x: asarray(2)).unstack().toarray().shape == (n, )
# composing functions works
assert s.map(lambda x: x * 2).map(lambda x: x * 2).unstack().shape == (100,
2)
assert s.map(lambda x: x * 2).map(lambda x: npones(
(2, 2))).unstack().shape == (n, 2, 2)
assert s.map(lambda x: npones((2, 2))).map(
lambda x: x * 2).unstack().shape == (n, 2, 2)
# check the result
assert allclose(
s.map(lambda x: x.sum(axis=1)).unstack().toarray(),
npones(100) * 2)
assert allclose(
s.map(lambda x: tile(x, (1, 2))).unstack().toarray(), npones((100, 4)))
with pytest.raises(ValueError):
s.map(lambda x: 2)
with pytest.raises(ValueError):
s.map(lambda x: None)
with pytest.raises(RuntimeError):
s.map(lambda x: 1 / 0)
def test_stacked_shape_inference(sc):
from numpy import ones as npones
a = ones((100, 2), sc)
a._rdd = a._rdd.partitionBy(2)
s = a.stack(5)
n = s.tordd().count()
# operations that preserve keys
assert s.map(lambda x: x * 2).unstack().shape == (100, 2)
assert s.map(lambda x: x.sum(axis=1)).unstack().shape == (100,)
assert s.map(lambda x: tile(x, (1, 2))).unstack().shape == (100, 4)
# operations that create new keys
assert s.map(lambda x: npones((2, 2))).unstack().shape == (n, 2, 2)
assert s.map(lambda x: x.sum(axis=0)).unstack().shape == (n, 2)
assert s.map(lambda x: asarray([2])).unstack().toarray().shape == (n, 1)
assert s.map(lambda x: asarray(2)).unstack().toarray().shape == (n,)
# composing functions works
assert s.map(lambda x: x * 2).map(lambda x: x * 2).unstack().shape == (100, 2)
assert s.map(lambda x: x * 2).map(lambda x: npones((2, 2))).unstack().shape == (n, 2, 2)
assert s.map(lambda x: npones((2, 2))).map(lambda x: x * 2).unstack().shape == (n, 2, 2)
# check the result
assert allclose(s.map(lambda x: x.sum(axis=1)).unstack().toarray(), npones(100) * 2)
assert allclose(s.map(lambda x: tile(x, (1, 2))).unstack().toarray(), npones((100, 4)))
with pytest.raises(ValueError):
s.map(lambda x: 2)
with pytest.raises(ValueError):
s.map(lambda x: None)
with pytest.raises(RuntimeError):
s.map(lambda x: 1/0)