def reduction_2d_test(da_func, darr, np_func, narr, use_dtype=True,
split_every=True):
assert_eq(da_func(darr), np_func(narr))
assert_eq(da_func(darr, keepdims=True), np_func(narr, keepdims=True))
assert_eq(da_func(darr, axis=0), np_func(narr, axis=0))
assert_eq(da_func(darr, axis=1), np_func(narr, axis=1))
assert_eq(da_func(darr, axis=-1), np_func(narr, axis=-1))
assert_eq(da_func(darr, axis=-2), np_func(narr, axis=-2))
assert_eq(da_func(darr, axis=1, keepdims=True),
np_func(narr, axis=1, keepdims=True))
assert_eq(da_func(darr, axis=(1, 0)), np_func(narr, axis=(1, 0)))
assert same_keys(da_func(darr, axis=1), da_func(darr, axis=1))
assert same_keys(da_func(darr, axis=(1, 0)), da_func(darr, axis=(1, 0)))
if use_dtype:
assert_eq(da_func(darr, dtype='f8'), np_func(narr, dtype='f8'))
assert_eq(da_func(darr, dtype='i8'), np_func(narr, dtype='i8'))
if split_every:
a1 = da_func(darr, split_every=4)
a2 = da_func(darr, split_every={0: 2, 1: 2})
assert same_keys(a1, a2)
assert_eq(a1, np_func(narr))
assert_eq(a2, np_func(narr))
assert_eq(da_func(darr, keepdims=True, split_every=4),
np_func(narr, keepdims=True))
assert_eq(da_func(darr, axis=0, split_every=2), np_func(narr, axis=0))
assert_eq(da_func(darr, axis=0, keepdims=True, split_every=2),
np_func(narr, axis=0, keepdims=True))
assert_eq(da_func(darr, axis=1, split_every=2), np_func(narr, axis=1))
assert_eq(da_func(darr, axis=1, keepdims=True, split_every=2),
np_func(narr, axis=1, keepdims=True))
def test_slicing_consistent_names():
x = np.arange(100).reshape((10, 10))
a = da.from_array(x, chunks=(5, 5))
assert same_keys(a[0], a[0])
assert same_keys(a[:, [1, 2, 3]], a[:, [1, 2, 3]])
assert same_keys(a[:, 5:2:-1], a[:, 5:2:-1])
assert same_keys(a[0, ...], a[0, ...])
assert same_keys(a[...], a[...])
assert same_keys(a[[1, 3, 5]], a[[1, 3, 5]])
assert same_keys(a[-11:11], a[:])
assert same_keys(a[-11:-9], a[:1])
assert same_keys(a[-1], a[9])
assert same_keys(a[0::-1], a[0:-11:-1])
def test_linalg_consistent_names():
m, n = 20, 10
mat = np.random.rand(m, n)
data = da.from_array(mat, chunks=(10, n), name='A')
q1, r1 = qr(data)
q2, r2 = qr(data)
assert same_keys(q1, q2)
assert same_keys(r1, r2)
u1, s1, v1 = svd(data)
u2, s2, v2 = svd(data)
assert same_keys(u1, u2)
assert same_keys(s1, s2)
assert same_keys(v1, v2)
def test_fromfunction():
def f(x, y):
return x + y
d = da.fromfunction(f, shape=(5, 5), chunks=(2, 2), dtype='f8')
assert_eq(d, np.fromfunction(f, shape=(5, 5)))
assert same_keys(d, da.fromfunction(f, shape=(5, 5), chunks=(2, 2), dtype='f8'))
def test_insert():
x = np.random.randint(10, size=(10, 10))
a = da.from_array(x, chunks=(5, 5))
y = np.random.randint(10, size=(5, 10))
b = da.from_array(y, chunks=(4, 4))
assert_eq(np.insert(x, 0, -1, axis=0), da.insert(a, 0, -1, axis=0))
assert_eq(np.insert(x, 3, -1, axis=-1), da.insert(a, 3, -1, axis=-1))
assert_eq(np.insert(x, 5, -1, axis=1), da.insert(a, 5, -1, axis=1))
assert_eq(np.insert(x, -1, -1, axis=-2), da.insert(a, -1, -1, axis=-2))
assert_eq(np.insert(x, [2, 3, 3], -1, axis=1),
da.insert(a, [2, 3, 3], -1, axis=1))
assert_eq(np.insert(x, [2, 3, 8, 8, -2, -2], -1, axis=0),
da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0))
assert_eq(np.insert(x, slice(1, 4), -1, axis=1),
da.insert(a, slice(1, 4), -1, axis=1))
assert_eq(np.insert(x, [2] * 3 + [5] * 2, y, axis=0),
da.insert(a, [2] * 3 + [5] * 2, b, axis=0))
assert_eq(np.insert(x, 0, y[0], axis=1),
da.insert(a, 0, b[0], axis=1))
assert same_keys(da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0),
da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0))
with pytest.raises(NotImplementedError):
da.insert(a, [4, 2], -1, axis=0)
with pytest.raises(IndexError):
da.insert(a, [3], -1, axis=2)
with pytest.raises(IndexError):
da.insert(a, [3], -1, axis=-3)
def test_tensordot():
x = np.arange(400).reshape((20, 20))
a = da.from_array(x, chunks=(5, 4))
y = np.arange(200).reshape((20, 10))
b = da.from_array(y, chunks=(4, 5))
for axes in [1, (1, 0)]:
assert_eq(da.tensordot(a, b, axes=axes), np.tensordot(x, y, axes=axes))
assert_eq(da.tensordot(x, b, axes=axes), np.tensordot(x, y, axes=axes))
assert_eq(da.tensordot(a, y, axes=axes), np.tensordot(x, y, axes=axes))
assert same_keys(da.tensordot(a, b, axes=(1, 0)),
da.tensordot(a, b, axes=(1, 0)))
with pytest.warns(None): # Increasing number of chunks warning
assert not same_keys(da.tensordot(a, b, axes=0),
da.tensordot(a, b, axes=1))
def test_overlap_internal():
x = np.arange(64).reshape((8, 8))
d = da.from_array(x, chunks=(4, 4))
g = overlap_internal(d, {0: 2, 1: 1})
result = g.compute(scheduler='sync')
assert g.chunks == ((6, 6), (5, 5))
expected = np.array([
[ 0, 1, 2, 3, 4, 3, 4, 5, 6, 7],
[ 8, 9, 10, 11, 12, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 51, 52, 53, 54, 55],
[56, 57, 58, 59, 60, 59, 60, 61, 62, 63]])
assert_eq(result, expected)
assert same_keys(overlap_internal(d, {0: 2, 1: 1}), g)
def test_squeeze(is_func, axis):
a = np.arange(10)[None, :, None, None]
d = da.from_array(a, chunks=(1, 3, 1, 1))
if is_func:
a_s = np.squeeze(a, axis=axis)
d_s = da.squeeze(d, axis=axis)
else:
a_s = a.squeeze(axis=axis)
d_s = d.squeeze(axis=axis)
assert_eq(d_s, a_s)
assert same_keys(d_s, da.squeeze(d, axis=axis))
if axis is None:
axis = tuple(range(a.ndim))
else:
axis = axis if isinstance(axis, tuple) else (axis,)
axis = tuple(i % a.ndim for i in axis)
axis = tuple(
i for i, c in enumerate(d.chunks) if i in axis and len(c) == 1
)
exp_d_s_chunks = tuple(
c for i, c in enumerate(d.chunks) if i not in axis
)
assert d_s.chunks == exp_d_s_chunks
def test_bincount_unspecified_minlength():
x = np.array([1, 1, 3, 7, 0])
d = da.from_array(x, chunks=2)
e = da.bincount(d)
assert_eq(e, np.bincount(x))
assert same_keys(da.bincount(d), e)
assert len(e.compute()) == 8 # shape is (nan,) so must compute for len()
def test_squeeze():
x = da.ones((10, 1), chunks=(3, 1))
assert_eq(x.squeeze(), x.compute().squeeze())
assert x.squeeze().chunks == ((3, 3, 3, 1),)
assert same_keys(x.squeeze(), x.squeeze())
def test_transpose():
x = np.arange(240).reshape((4, 6, 10))
d = da.from_array(x, (2, 3, 4))
assert_eq(d.transpose((2, 0, 1)),
x.transpose((2, 0, 1)))
assert same_keys(d.transpose((2, 0, 1)), d.transpose((2, 0, 1)))
assert_eq(d.transpose(2, 0, 1),
x.transpose(2, 0, 1))
assert same_keys(d.transpose(2, 0, 1), d.transpose(2, 0, 1))
with pytest.raises(ValueError):
d.transpose(1, 2)
with pytest.raises(ValueError):
d.transpose((1, 2))
def reduction_1d_test(da_func, darr, np_func, narr, use_dtype=True, split_every=True):
assert_eq(da_func(darr), np_func(narr))
assert_eq(da_func(darr, keepdims=True), np_func(narr, keepdims=True))
assert same_keys(da_func(darr), da_func(darr))
assert same_keys(da_func(darr, keepdims=True), da_func(darr, keepdims=True))
if use_dtype:
assert_eq(da_func(darr, dtype='f8'), np_func(narr, dtype='f8'))
assert_eq(da_func(darr, dtype='i8'), np_func(narr, dtype='i8'))
assert same_keys(da_func(darr, dtype='i8'), da_func(darr, dtype='i8'))
if split_every:
a1 = da_func(darr, split_every=2)
a2 = da_func(darr, split_every={0: 2})
assert same_keys(a1, a2)
assert_eq(a1, np_func(narr))
assert_eq(a2, np_func(narr))
assert_eq(da_func(darr, keepdims=True, split_every=2),
np_func(narr, keepdims=True))
def test_topk():
x = np.array([5, 2, 1, 6])
d = da.from_array(x, chunks=2)
e = da.topk(2, d)
assert e.chunks == ((2,),)
assert_eq(e, np.sort(x)[-1:-3:-1])
assert same_keys(da.topk(2, d), e)
def test_bincount():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
e = da.bincount(d, minlength=6)
assert_eq(e, np.bincount(x, minlength=6))
assert same_keys(da.bincount(d, minlength=6), e)
assert da.bincount(d, minlength=6).name != da.bincount(d, minlength=7).name
assert da.bincount(d, minlength=6).name == da.bincount(d, minlength=6).name
def test_bincount_with_weights():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
weights = np.array([1, 2, 1, 0.5, 1])
dweights = da.from_array(weights, chunks=2)
e = da.bincount(d, weights=dweights, minlength=6)
assert_eq(e, np.bincount(x, weights=dweights, minlength=6))
assert same_keys(da.bincount(d, weights=dweights, minlength=6), e)
def test_percentile():
d = da.ones((16,), chunks=(4,))
assert_eq(da.percentile(d, [0, 50, 100]),
np.array([1, 1, 1], dtype=d.dtype))
x = np.array([0, 0, 5, 5, 5, 5, 20, 20])
d = da.from_array(x, chunks=(3,))
result = da.percentile(d, [0, 50, 100])
assert_eq(da.percentile(d, [0, 50, 100]),
np.array([0, 5, 20], dtype=result.dtype))
assert same_keys(da.percentile(d, [0, 50, 100]),
da.percentile(d, [0, 50, 100]))
assert not same_keys(da.percentile(d, [0, 50, 100]),
da.percentile(d, [0, 50]))
x = np.array(['a', 'a', 'd', 'd', 'd', 'e'])
d = da.from_array(x, chunks=(3,))
assert_eq(da.percentile(d, [0, 50, 100]),
np.array(['a', 'd', 'e'], dtype=x.dtype))
def test_take():
x = np.arange(400).reshape((20, 20))
a = da.from_array(x, chunks=(5, 5))
assert_eq(np.take(x, 3, axis=0), da.take(a, 3, axis=0))
assert_eq(np.take(x, [3, 4, 5], axis=-1), da.take(a, [3, 4, 5], axis=-1))
with pytest.raises(ValueError):
da.take(a, 3, axis=2)
assert same_keys(da.take(a, [3, 4, 5], axis=-1),
da.take(a, [3, 4, 5], axis=-1))
def reduction_1d_test(da_func, darr, np_func, narr, use_dtype=True, split_every=True):
assert_eq(da_func(darr), np_func(narr))
assert_eq(
da_func(narr), np_func(narr)
) # Ensure Dask reductions work with NumPy arrays
assert_eq(da_func(darr, keepdims=True), np_func(narr, keepdims=True))
assert_eq(da_func(darr, axis=()), np_func(narr, axis=()))
assert same_keys(da_func(darr), da_func(darr))
assert same_keys(da_func(darr, keepdims=True), da_func(darr, keepdims=True))
if use_dtype:
assert_eq(da_func(darr, dtype="f8"), np_func(narr, dtype="f8"))
assert_eq(da_func(darr, dtype="i8"), np_func(narr, dtype="i8"))
assert same_keys(da_func(darr, dtype="i8"), da_func(darr, dtype="i8"))
if split_every:
a1 = da_func(darr, split_every=2)
a2 = da_func(darr, split_every={0: 2})
assert same_keys(a1, a2)
assert_eq(a1, np_func(narr))
assert_eq(a2, np_func(narr))
assert_eq(
da_func(darr, keepdims=True, split_every=2), np_func(narr, keepdims=True)
)
def test_fromfunction(func, dtype, kwargs):
a = np.fromfunction(func, shape=(5, 5), dtype=dtype, **kwargs)
d = da.fromfunction(
func, shape=(5, 5), chunks=(2, 2), dtype=dtype, **kwargs
)
assert_eq(d, a)
d2 = da.fromfunction(
func, shape=(5, 5), chunks=(2, 2), dtype=dtype, **kwargs
)
assert same_keys(d, d2)
def test_fromfunction(func, dtype, kwargs):
a = np.fromfunction(func, shape=(5, 5), dtype=dtype, **kwargs)
d = da.fromfunction(
func, shape=(5, 5), chunks=(2, 2), dtype=dtype, **kwargs
)
assert_eq(d, a)
d2 = da.fromfunction(
func, shape=(5, 5), chunks=(2, 2), dtype=dtype, **kwargs
)
assert same_keys(d, d2)
def test_histogram():
# Test for normal, flattened input
n = 100
v = da.random.random(n, chunks=10)
bins = np.arange(0, 1.01, 0.01)
(a1, b1) = da.histogram(v, bins=bins)
(a2, b2) = np.histogram(v, bins=bins)
# Check if the sum of the bins equals the number of samples
assert a2.sum(axis=0) == n
assert a1.sum(axis=0) == n
assert_eq(a1, a2)
assert same_keys(da.histogram(v, bins=bins)[0], a1)
def test_histogram():
# Test for normal, flattened input
n = 100
v = da.random.random(n, chunks=10)
bins = np.arange(0, 1.01, 0.01)
(a1, b1) = da.histogram(v, bins=bins)
(a2, b2) = np.histogram(v, bins=bins)
# Check if the sum of the bins equals the number of samples
assert a2.sum(axis=0) == n
assert a1.sum(axis=0) == n
assert_eq(a1, a2)
assert same_keys(da.histogram(v, bins=bins)[0], a1)
def test_ghost():
x = np.arange(64).reshape((8, 8))
d = da.from_array(x, chunks=(4, 4))
g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'reflect'})
assert g.chunks == ((8, 8), (6, 6))
expected = np.array(
[[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[0, 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 7],
[8, 8, 9, 10, 11, 12, 11, 12, 13, 14, 15, 15],
[16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23],
[24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31],
[32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39],
[40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47],
[16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23],
[24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31],
[32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39],
[40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47],
[48, 48, 49, 50, 51, 52, 51, 52, 53, 54, 55, 55],
[56, 56, 57, 58, 59, 60, 59, 60, 61, 62, 63, 63],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]])
assert_eq(g, expected)
assert same_keys(
g, ghost(d, depth={
0: 2,
1: 1
}, boundary={
0: 100,
1: 'reflect'
}))
g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'none'})
expected = np.array([[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[0, 1, 2, 3, 4, 3, 4, 5, 6, 7],
[8, 9, 10, 11, 12, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 51, 52, 53, 54, 55],
[56, 57, 58, 59, 60, 59, 60, 61, 62, 63],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100]])
assert_eq(g, expected)
assert g.chunks == ((8, 8), (5, 5))
def test_percentile(method):
d = da.ones((16, ), chunks=(4, ))
qs = [0, 50, 100]
assert_eq(da.percentile(d, qs, method=method),
np.array([1, 1, 1], dtype=d.dtype))
x = np.array([0, 0, 5, 5, 5, 5, 20, 20])
d = da.from_array(x, chunks=(3, ))
result = da.percentile(d, qs, method=method)
assert_eq(result, np.array([0, 5, 20], dtype=result.dtype))
assert same_keys(da.percentile(d, qs, method=method),
da.percentile(d, qs, method=method))
assert not same_keys(da.percentile(d, qs, method=method),
da.percentile(d, [0, 50], method=method))
if method != 'tdigest':
x = np.array(['a', 'a', 'd', 'd', 'd', 'e'])
d = da.from_array(x, chunks=(3, ))
assert_eq(da.percentile(d, [0, 50, 100]),
np.array(['a', 'd', 'e'], dtype=x.dtype))
def reduction_1d_test(da_func,
darr,
np_func,
narr,
use_dtype=True,
split_every=True):
assert_eq(da_func(darr), np_func(narr))
assert_eq(da_func(darr, keepdims=True), np_func(narr, keepdims=True))
assert same_keys(da_func(darr), da_func(darr))
assert same_keys(da_func(darr, keepdims=True), da_func(darr,
keepdims=True))
if use_dtype:
assert_eq(da_func(darr, dtype='f8'), np_func(narr, dtype='f8'))
assert_eq(da_func(darr, dtype='i8'), np_func(narr, dtype='i8'))
assert same_keys(da_func(darr, dtype='i8'), da_func(darr, dtype='i8'))
if split_every:
a1 = da_func(darr, split_every=2)
a2 = da_func(darr, split_every={0: 2})
assert same_keys(a1, a2)
assert_eq(a1, np_func(narr))
assert_eq(a2, np_func(narr))
assert_eq(da_func(darr, keepdims=True, split_every=2),
np_func(narr, keepdims=True))
def reduction_2d_test(da_func,
darr,
np_func,
narr,
use_dtype=True,
split_every=True):
assert_eq(da_func(darr), np_func(narr))
assert_eq(da_func(darr, keepdims=True), np_func(narr, keepdims=True))
assert_eq(da_func(darr, axis=0), np_func(narr, axis=0))
assert_eq(da_func(darr, axis=1), np_func(narr, axis=1))
assert_eq(da_func(darr, axis=-1), np_func(narr, axis=-1))
assert_eq(da_func(darr, axis=-2), np_func(narr, axis=-2))
assert_eq(da_func(darr, axis=1, keepdims=True),
np_func(narr, axis=1, keepdims=True))
assert_eq(da_func(darr, axis=(1, 0)), np_func(narr, axis=(1, 0)))
assert same_keys(da_func(darr, axis=1), da_func(darr, axis=1))
assert same_keys(da_func(darr, axis=(1, 0)), da_func(darr, axis=(1, 0)))
if use_dtype:
assert_eq(da_func(darr, dtype='f8'), np_func(narr, dtype='f8'))
assert_eq(da_func(darr, dtype='i8'), np_func(narr, dtype='i8'))
if split_every:
a1 = da_func(darr, split_every=4)
a2 = da_func(darr, split_every={0: 2, 1: 2})
assert same_keys(a1, a2)
assert_eq(a1, np_func(narr))
assert_eq(a2, np_func(narr))
assert_eq(da_func(darr, keepdims=True, split_every=4),
np_func(narr, keepdims=True))
assert_eq(da_func(darr, axis=0, split_every=2), np_func(narr, axis=0))
assert_eq(da_func(darr, axis=0, keepdims=True, split_every=2),
np_func(narr, axis=0, keepdims=True))
assert_eq(da_func(darr, axis=1, split_every=2), np_func(narr, axis=1))
assert_eq(da_func(darr, axis=1, keepdims=True, split_every=2),
np_func(narr, axis=1, keepdims=True))
def test_percentile(method):
d = da.ones((16,), chunks=(4,))
qs = [0, 50, 100]
assert_eq(da.percentile(d, qs, method=method),
np.array([1, 1, 1], dtype=d.dtype))
x = np.array([0, 0, 5, 5, 5, 5, 20, 20])
d = da.from_array(x, chunks=(3,))
result = da.percentile(d, qs, method=method)
assert_eq(result,
np.array([0, 5, 20], dtype=result.dtype))
assert same_keys(da.percentile(d, qs, method=method),
da.percentile(d, qs, method=method))
assert not same_keys(da.percentile(d, qs, method=method),
da.percentile(d, [0, 50], method=method))
if method != 'tdigest':
x = np.array(['a', 'a', 'd', 'd', 'd', 'e'])
d = da.from_array(x, chunks=(3,))
assert_eq(da.percentile(d, [0, 50, 100]),
np.array(['a', 'd', 'e'], dtype=x.dtype))
def test_percentile_with_categoricals():
try:
import pandas as pd
except ImportError:
return
x0 = pd.Categorical(["Alice", "Bob", "Charlie", "Dennis", "Alice", "Alice"])
x1 = pd.Categorical(["Alice", "Bob", "Charlie", "Dennis", "Alice", "Alice"])
dsk = {("x", 0): x0, ("x", 1): x1}
x = da.Array(dsk, "x", chunks=((6, 6),))
p = da.percentile(x, [50])
assert (p.compute().categories == x0.categories).all()
assert (p.compute().codes == [0]).all()
assert same_keys(da.percentile(x, [50]), da.percentile(x, [50]))
def test_overlap():
x = np.arange(64).reshape((8, 8))
d = da.from_array(x, chunks=(4, 4))
g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'reflect'})
assert g.chunks == ((8, 8), (6, 6))
expected = np.array(
[[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[ 0, 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 7],
[ 8, 8, 9, 10, 11, 12, 11, 12, 13, 14, 15, 15],
[ 16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23],
[ 24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31],
[ 32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39],
[ 40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47],
[ 16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23],
[ 24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31],
[ 32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39],
[ 40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47],
[ 48, 48, 49, 50, 51, 52, 51, 52, 53, 54, 55, 55],
[ 56, 56, 57, 58, 59, 60, 59, 60, 61, 62, 63, 63],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]])
assert_eq(g, expected)
assert same_keys(g, overlap(d, depth={0: 2, 1: 1},
boundary={0: 100, 1: 'reflect'}))
g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'none'})
expected = np.array(
[[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[ 0, 1, 2, 3, 4, 3, 4, 5, 6, 7],
[ 8, 9, 10, 11, 12, 11, 12, 13, 14, 15],
[ 16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[ 24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[ 32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[ 40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[ 16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[ 24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[ 32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[ 40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[ 48, 49, 50, 51, 52, 51, 52, 53, 54, 55],
[ 56, 57, 58, 59, 60, 59, 60, 61, 62, 63],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100]])
assert_eq(g, expected)
assert g.chunks == ((8, 8), (5, 5))
def test_percentile_with_categoricals():
try:
import pandas as pd
except ImportError:
return
x0 = pd.Categorical(['Alice', 'Bob', 'Charlie', 'Dennis', 'Alice', 'Alice'])
x1 = pd.Categorical(['Alice', 'Bob', 'Charlie', 'Dennis', 'Alice', 'Alice'])
dsk = {('x', 0): x0, ('x', 1): x1}
x = da.Array(dsk, 'x', chunks=((6, 6),))
p = da.percentile(x, [50])
assert (p.compute().categories == x0.categories).all()
assert (p.compute().codes == [0]).all()
assert same_keys(da.percentile(x, [50]),
da.percentile(x, [50]))
def test_percentile_with_categoricals():
try:
import pandas as pd
except ImportError:
return
x0 = pd.Categorical(
['Alice', 'Bob', 'Charlie', 'Dennis', 'Alice', 'Alice'])
x1 = pd.Categorical(
['Alice', 'Bob', 'Charlie', 'Dennis', 'Alice', 'Alice'])
dsk = {('x', 0): x0, ('x', 1): x1}
x = da.Array(dsk, 'x', chunks=((6, 6), ))
p = da.percentile(x, [50])
assert (p.compute().categories == x0.categories).all()
assert (p.compute().codes == [0]).all()
assert same_keys(da.percentile(x, [50]), da.percentile(x, [50]))
def test_overlap_internal_asymmetric_small():
x = np.arange(32).reshape((2, 16))
d = da.from_array(x, chunks=(2, 4))
result = overlap_internal(d, {0: (0, 0), 1: (1, 1)})
assert result.chunks == ((2, ), (5, 6, 6, 5))
expected = np.array([[
0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 8, 7, 8, 9, 10, 11, 12, 11, 12, 13, 14,
15
],
[
16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 24, 23,
24, 25, 26, 27, 28, 27, 28, 29, 30, 31
]])
assert_eq(result, expected)
assert same_keys(overlap_internal(d, {0: (0, 0), 1: (1, 1)}), result)
def test_overlap_internal_asymmetric():
x = np.arange(64).reshape((8, 8))
d = da.from_array(x, chunks=(4, 4))
result = overlap_internal(d, {0: (2, 0), 1: (1, 0)})
assert result.chunks == ((6, 4), (5, 4))
expected = np.array([[0, 1, 2, 3, 3, 4, 5, 6, 7],
[8, 9, 10, 11, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 27, 28, 29, 30, 31],
[16, 17, 18, 19, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 51, 52, 53, 54, 55],
[56, 57, 58, 59, 59, 60, 61, 62, 63]])
assert_eq(result, expected)
assert same_keys(overlap_internal(d, {0: (2, 0), 1: (1, 0)}), result)
def test_percentile():
d = da.from_array(cupy.ones((16,)), chunks=(4,))
qs = np.array([0, 50, 100])
assert_eq(
da.percentile(d, qs, interpolation="midpoint"),
np.array([1, 1, 1], dtype=d.dtype),
)
x = cupy.array([0, 0, 5, 5, 5, 5, 20, 20])
d = da.from_array(x, chunks=(3,))
result = da.percentile(d, qs, interpolation="midpoint")
assert_eq(result, np.array([0, 5, 20], dtype=result.dtype))
assert not same_keys(
da.percentile(d, qs, interpolation="midpoint"),
da.percentile(d, [0, 50], interpolation="midpoint"),
)
def test_squeeze(is_func, axis):
a = np.arange(10)[None, :, None, None]
d = da.from_array(a, chunks=(1, 3, 1, 1))
if is_func:
a_s = np.squeeze(a, axis=axis)
d_s = da.squeeze(d, axis=axis)
else:
a_s = a.squeeze(axis=axis)
d_s = d.squeeze(axis=axis)
assert_eq(d_s, a_s)
assert same_keys(d_s, da.squeeze(d, axis=axis))
if axis is None:
axis = tuple(range(a.ndim))
else:
axis = axis if isinstance(axis, tuple) else (axis,)
axis = tuple(i % a.ndim for i in axis)
axis = tuple(i for i, c in enumerate(d.chunks) if i in axis and len(c) == 1)
exp_d_s_chunks = tuple(c for i, c in enumerate(d.chunks) if i not in axis)
assert d_s.chunks == exp_d_s_chunks
def test_ghost_internal():
x = np.arange(64).reshape((8, 8))
d = da.from_array(x, chunks=(4, 4))
g = ghost_internal(d, {0: 2, 1: 1})
result = g.compute(get=get)
assert g.chunks == ((6, 6), (5, 5))
expected = np.array([[0, 1, 2, 3, 4, 3, 4, 5, 6, 7],
[8, 9, 10, 11, 12, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 51, 52, 53, 54, 55],
[56, 57, 58, 59, 60, 59, 60, 61, 62, 63]])
assert_eq(result, expected)
assert same_keys(ghost_internal(d, {0: 2, 1: 1}), g)
def test_insert():
x = np.random.randint(10, size=(10, 10))
a = da.from_array(x, chunks=(5, 5))
y = np.random.randint(10, size=(5, 10))
b = da.from_array(y, chunks=(4, 4))
assert_eq(np.insert(x, 0, -1, axis=0), da.insert(a, 0, -1, axis=0))
assert_eq(np.insert(x, 3, -1, axis=-1), da.insert(a, 3, -1, axis=-1))
assert_eq(np.insert(x, 5, -1, axis=1), da.insert(a, 5, -1, axis=1))
assert_eq(np.insert(x, -1, -1, axis=-2), da.insert(a, -1, -1, axis=-2))
assert_eq(np.insert(x, [2, 3, 3], -1, axis=1),
da.insert(a, [2, 3, 3], -1, axis=1))
assert_eq(
np.insert(x, [2, 3, 8, 8, -2, -2], -1, axis=0),
da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0),
)
assert_eq(np.insert(x, slice(1, 4), -1, axis=1),
da.insert(a, slice(1, 4), -1, axis=1))
assert_eq(
np.insert(x, [2] * 3 + [5] * 2, y, axis=0),
da.insert(a, [2] * 3 + [5] * 2, b, axis=0),
)
assert_eq(np.insert(x, 0, y[0], axis=1), da.insert(a, 0, b[0], axis=1))
assert same_keys(
da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0),
da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0),
)
with pytest.raises(NotImplementedError):
da.insert(a, [4, 2], -1, axis=0)
with pytest.raises(AxisError):
da.insert(a, [3], -1, axis=2)
with pytest.raises(AxisError):
da.insert(a, [3], -1, axis=-3)
def test_overlap_internal():
x = cupy.arange(64).reshape((8, 8))
d = da.from_array(x, chunks=(4, 4), asarray=False)
g = da.overlap.overlap_internal(d, {0: 2, 1: 1})
assert g.chunks == ((6, 6), (5, 5))
expected = np.array([
[0, 1, 2, 3, 4, 3, 4, 5, 6, 7],
[8, 9, 10, 11, 12, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 51, 52, 53, 54, 55],
[56, 57, 58, 59, 60, 59, 60, 61, 62, 63],
])
assert_eq(g, expected)
assert same_keys(da.overlap.overlap_internal(d, {0: 2, 1: 1}), g)
def test_slicing_consistent_names():
x = np.arange(100).reshape((10, 10))
a = da.from_array(x, chunks=(5, 5))
assert same_keys(a[0], a[0])
assert same_keys(a[:, [1, 2, 3]], a[:, [1, 2, 3]])
assert same_keys(a[:, 5:2:-1], a[:, 5:2:-1])
def test_overlap():
x = np.arange(64).reshape((8, 8))
d = da.from_array(x, chunks=(4, 4))
g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: "reflect"})
assert g.chunks == ((8, 8), (6, 6))
expected = np.array([
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[0, 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 7],
[8, 8, 9, 10, 11, 12, 11, 12, 13, 14, 15, 15],
[16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23],
[24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31],
[32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39],
[40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47],
[16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23],
[24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31],
[32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39],
[40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47],
[48, 48, 49, 50, 51, 52, 51, 52, 53, 54, 55, 55],
[56, 56, 57, 58, 59, 60, 59, 60, 61, 62, 63, 63],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
])
assert_eq(g, expected)
assert same_keys(
g, overlap(d, depth={
0: 2,
1: 1
}, boundary={
0: 100,
1: "reflect"
}))
u_depth = np.uint16([2, 1])
u_depth = {k: v for k, v in enumerate(u_depth)}
g = overlap(d, depth=u_depth, boundary={0: 100, 1: "reflect"})
assert g.chunks == ((8, 8), (6, 6))
assert_eq(g, expected)
assert same_keys(
g, overlap(d, depth={
0: 2,
1: 1
}, boundary={
0: 100,
1: "reflect"
}))
g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: "none"})
expected = np.array([
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[0, 1, 2, 3, 4, 3, 4, 5, 6, 7],
[8, 9, 10, 11, 12, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 51, 52, 53, 54, 55],
[56, 57, 58, 59, 60, 59, 60, 61, 62, 63],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
])
assert_eq(g, expected)
assert g.chunks == ((8, 8), (5, 5))
u_depth = np.uint16([2, 1])
u_depth = {k: v for k, v in enumerate(u_depth)}
g = overlap(d, depth=u_depth, boundary={0: 100, 1: "none"})
assert_eq(g, expected)
assert g.chunks == ((8, 8), (5, 5))
def test_slicing_consistent_names_after_normalization():
x = da.zeros(10, chunks=(5,))
assert same_keys(x[0:], x[:10])
assert same_keys(x[0:], x[0:10])
assert same_keys(x[0:], x[0:10:1])
assert same_keys(x[:], x[0:10:1])
def test_fft_consistent_names(funcname):
da_fft = getattr(da.fft, funcname)
assert same_keys(da_fft(darr, 5), da_fft(darr, 5))
assert same_keys(da_fft(darr2, 5, axis=0), da_fft(darr2, 5, axis=0))
assert not same_keys(da_fft(darr, 5), da_fft(darr, 13))
def test_percentile_tokenize():
d = da.from_array(cupy.ones((16,)), chunks=(4,))
qs = np.array([0, 50, 100])
assert same_keys(da.percentile(d, qs), da.percentile(d, qs))
def test_bincount():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
e = da.bincount(d, minlength=6)
assert_eq(e, np.bincount(x, minlength=6))
assert same_keys(da.bincount(d, minlength=6), e)
def test_fft_consistent_names(funcname):
da_fft = getattr(da.fft, funcname)
assert same_keys(da_fft(darr, 5), da_fft(darr, 5))
assert same_keys(da_fft(darr2, 5, axis=0), da_fft(darr2, 5, axis=0))
assert not same_keys(da_fft(darr, 5), da_fft(darr, 13))
