def reduction_2d_test(da_func, darr, np_func, narr, use_dtype=True, split_every=True): assert_eq(da_func(darr), np_func(narr)) assert_eq(da_func(darr, keepdims=True), np_func(narr, keepdims=True)) assert_eq(da_func(darr, axis=0), np_func(narr, axis=0)) assert_eq(da_func(darr, axis=1), np_func(narr, axis=1)) assert_eq(da_func(darr, axis=-1), np_func(narr, axis=-1)) assert_eq(da_func(darr, axis=-2), np_func(narr, axis=-2)) assert_eq(da_func(darr, axis=1, keepdims=True), np_func(narr, axis=1, keepdims=True)) assert_eq(da_func(darr, axis=(1, 0)), np_func(narr, axis=(1, 0))) assert same_keys(da_func(darr, axis=1), da_func(darr, axis=1)) assert same_keys(da_func(darr, axis=(1, 0)), da_func(darr, axis=(1, 0))) if use_dtype: assert_eq(da_func(darr, dtype='f8'), np_func(narr, dtype='f8')) assert_eq(da_func(darr, dtype='i8'), np_func(narr, dtype='i8')) if split_every: a1 = da_func(darr, split_every=4) a2 = da_func(darr, split_every={0: 2, 1: 2}) assert same_keys(a1, a2) assert_eq(a1, np_func(narr)) assert_eq(a2, np_func(narr)) assert_eq(da_func(darr, keepdims=True, split_every=4), np_func(narr, keepdims=True)) assert_eq(da_func(darr, axis=0, split_every=2), np_func(narr, axis=0)) assert_eq(da_func(darr, axis=0, keepdims=True, split_every=2), np_func(narr, axis=0, keepdims=True)) assert_eq(da_func(darr, axis=1, split_every=2), np_func(narr, axis=1)) assert_eq(da_func(darr, axis=1, keepdims=True, split_every=2), np_func(narr, axis=1, keepdims=True))
def test_slicing_consistent_names(): x = np.arange(100).reshape((10, 10)) a = da.from_array(x, chunks=(5, 5)) assert same_keys(a[0], a[0]) assert same_keys(a[:, [1, 2, 3]], a[:, [1, 2, 3]]) assert same_keys(a[:, 5:2:-1], a[:, 5:2:-1]) assert same_keys(a[0, ...], a[0, ...]) assert same_keys(a[...], a[...]) assert same_keys(a[[1, 3, 5]], a[[1, 3, 5]]) assert same_keys(a[-11:11], a[:]) assert same_keys(a[-11:-9], a[:1]) assert same_keys(a[-1], a[9]) assert same_keys(a[0::-1], a[0:-11:-1])
def test_linalg_consistent_names(): m, n = 20, 10 mat = np.random.rand(m, n) data = da.from_array(mat, chunks=(10, n), name='A') q1, r1 = qr(data) q2, r2 = qr(data) assert same_keys(q1, q2) assert same_keys(r1, r2) u1, s1, v1 = svd(data) u2, s2, v2 = svd(data) assert same_keys(u1, u2) assert same_keys(s1, s2) assert same_keys(v1, v2)
def test_fromfunction(): def f(x, y): return x + y d = da.fromfunction(f, shape=(5, 5), chunks=(2, 2), dtype='f8') assert_eq(d, np.fromfunction(f, shape=(5, 5))) assert same_keys(d, da.fromfunction(f, shape=(5, 5), chunks=(2, 2), dtype='f8'))
def test_insert(): x = np.random.randint(10, size=(10, 10)) a = da.from_array(x, chunks=(5, 5)) y = np.random.randint(10, size=(5, 10)) b = da.from_array(y, chunks=(4, 4)) assert_eq(np.insert(x, 0, -1, axis=0), da.insert(a, 0, -1, axis=0)) assert_eq(np.insert(x, 3, -1, axis=-1), da.insert(a, 3, -1, axis=-1)) assert_eq(np.insert(x, 5, -1, axis=1), da.insert(a, 5, -1, axis=1)) assert_eq(np.insert(x, -1, -1, axis=-2), da.insert(a, -1, -1, axis=-2)) assert_eq(np.insert(x, [2, 3, 3], -1, axis=1), da.insert(a, [2, 3, 3], -1, axis=1)) assert_eq(np.insert(x, [2, 3, 8, 8, -2, -2], -1, axis=0), da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0)) assert_eq(np.insert(x, slice(1, 4), -1, axis=1), da.insert(a, slice(1, 4), -1, axis=1)) assert_eq(np.insert(x, [2] * 3 + [5] * 2, y, axis=0), da.insert(a, [2] * 3 + [5] * 2, b, axis=0)) assert_eq(np.insert(x, 0, y[0], axis=1), da.insert(a, 0, b[0], axis=1)) assert same_keys(da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0), da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0)) with pytest.raises(NotImplementedError): da.insert(a, [4, 2], -1, axis=0) with pytest.raises(IndexError): da.insert(a, [3], -1, axis=2) with pytest.raises(IndexError): da.insert(a, [3], -1, axis=-3)
def test_tensordot(): x = np.arange(400).reshape((20, 20)) a = da.from_array(x, chunks=(5, 4)) y = np.arange(200).reshape((20, 10)) b = da.from_array(y, chunks=(4, 5)) for axes in [1, (1, 0)]: assert_eq(da.tensordot(a, b, axes=axes), np.tensordot(x, y, axes=axes)) assert_eq(da.tensordot(x, b, axes=axes), np.tensordot(x, y, axes=axes)) assert_eq(da.tensordot(a, y, axes=axes), np.tensordot(x, y, axes=axes)) assert same_keys(da.tensordot(a, b, axes=(1, 0)), da.tensordot(a, b, axes=(1, 0))) with pytest.warns(None): # Increasing number of chunks warning assert not same_keys(da.tensordot(a, b, axes=0), da.tensordot(a, b, axes=1))
def test_overlap_internal(): x = np.arange(64).reshape((8, 8)) d = da.from_array(x, chunks=(4, 4)) g = overlap_internal(d, {0: 2, 1: 1}) result = g.compute(scheduler='sync') assert g.chunks == ((6, 6), (5, 5)) expected = np.array([ [ 0, 1, 2, 3, 4, 3, 4, 5, 6, 7], [ 8, 9, 10, 11, 12, 11, 12, 13, 14, 15], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [48, 49, 50, 51, 52, 51, 52, 53, 54, 55], [56, 57, 58, 59, 60, 59, 60, 61, 62, 63]]) assert_eq(result, expected) assert same_keys(overlap_internal(d, {0: 2, 1: 1}), g)
def test_squeeze(is_func, axis): a = np.arange(10)[None, :, None, None] d = da.from_array(a, chunks=(1, 3, 1, 1)) if is_func: a_s = np.squeeze(a, axis=axis) d_s = da.squeeze(d, axis=axis) else: a_s = a.squeeze(axis=axis) d_s = d.squeeze(axis=axis) assert_eq(d_s, a_s) assert same_keys(d_s, da.squeeze(d, axis=axis)) if axis is None: axis = tuple(range(a.ndim)) else: axis = axis if isinstance(axis, tuple) else (axis,) axis = tuple(i % a.ndim for i in axis) axis = tuple( i for i, c in enumerate(d.chunks) if i in axis and len(c) == 1 ) exp_d_s_chunks = tuple( c for i, c in enumerate(d.chunks) if i not in axis ) assert d_s.chunks == exp_d_s_chunks
def test_bincount_unspecified_minlength(): x = np.array([1, 1, 3, 7, 0]) d = da.from_array(x, chunks=2) e = da.bincount(d) assert_eq(e, np.bincount(x)) assert same_keys(da.bincount(d), e) assert len(e.compute()) == 8 # shape is (nan,) so must compute for len()
def test_squeeze(): x = da.ones((10, 1), chunks=(3, 1)) assert_eq(x.squeeze(), x.compute().squeeze()) assert x.squeeze().chunks == ((3, 3, 3, 1),) assert same_keys(x.squeeze(), x.squeeze())
def test_transpose(): x = np.arange(240).reshape((4, 6, 10)) d = da.from_array(x, (2, 3, 4)) assert_eq(d.transpose((2, 0, 1)), x.transpose((2, 0, 1))) assert same_keys(d.transpose((2, 0, 1)), d.transpose((2, 0, 1))) assert_eq(d.transpose(2, 0, 1), x.transpose(2, 0, 1)) assert same_keys(d.transpose(2, 0, 1), d.transpose(2, 0, 1)) with pytest.raises(ValueError): d.transpose(1, 2) with pytest.raises(ValueError): d.transpose((1, 2))
def reduction_1d_test(da_func, darr, np_func, narr, use_dtype=True, split_every=True): assert_eq(da_func(darr), np_func(narr)) assert_eq(da_func(darr, keepdims=True), np_func(narr, keepdims=True)) assert same_keys(da_func(darr), da_func(darr)) assert same_keys(da_func(darr, keepdims=True), da_func(darr, keepdims=True)) if use_dtype: assert_eq(da_func(darr, dtype='f8'), np_func(narr, dtype='f8')) assert_eq(da_func(darr, dtype='i8'), np_func(narr, dtype='i8')) assert same_keys(da_func(darr, dtype='i8'), da_func(darr, dtype='i8')) if split_every: a1 = da_func(darr, split_every=2) a2 = da_func(darr, split_every={0: 2}) assert same_keys(a1, a2) assert_eq(a1, np_func(narr)) assert_eq(a2, np_func(narr)) assert_eq(da_func(darr, keepdims=True, split_every=2), np_func(narr, keepdims=True))
def test_topk(): x = np.array([5, 2, 1, 6]) d = da.from_array(x, chunks=2) e = da.topk(2, d) assert e.chunks == ((2,),) assert_eq(e, np.sort(x)[-1:-3:-1]) assert same_keys(da.topk(2, d), e)
def test_bincount(): x = np.array([2, 1, 5, 2, 1]) d = da.from_array(x, chunks=2) e = da.bincount(d, minlength=6) assert_eq(e, np.bincount(x, minlength=6)) assert same_keys(da.bincount(d, minlength=6), e) assert da.bincount(d, minlength=6).name != da.bincount(d, minlength=7).name assert da.bincount(d, minlength=6).name == da.bincount(d, minlength=6).name
def test_bincount_with_weights(): x = np.array([2, 1, 5, 2, 1]) d = da.from_array(x, chunks=2) weights = np.array([1, 2, 1, 0.5, 1]) dweights = da.from_array(weights, chunks=2) e = da.bincount(d, weights=dweights, minlength=6) assert_eq(e, np.bincount(x, weights=dweights, minlength=6)) assert same_keys(da.bincount(d, weights=dweights, minlength=6), e)
def test_percentile(): d = da.ones((16,), chunks=(4,)) assert_eq(da.percentile(d, [0, 50, 100]), np.array([1, 1, 1], dtype=d.dtype)) x = np.array([0, 0, 5, 5, 5, 5, 20, 20]) d = da.from_array(x, chunks=(3,)) result = da.percentile(d, [0, 50, 100]) assert_eq(da.percentile(d, [0, 50, 100]), np.array([0, 5, 20], dtype=result.dtype)) assert same_keys(da.percentile(d, [0, 50, 100]), da.percentile(d, [0, 50, 100])) assert not same_keys(da.percentile(d, [0, 50, 100]), da.percentile(d, [0, 50])) x = np.array(['a', 'a', 'd', 'd', 'd', 'e']) d = da.from_array(x, chunks=(3,)) assert_eq(da.percentile(d, [0, 50, 100]), np.array(['a', 'd', 'e'], dtype=x.dtype))
def test_take(): x = np.arange(400).reshape((20, 20)) a = da.from_array(x, chunks=(5, 5)) assert_eq(np.take(x, 3, axis=0), da.take(a, 3, axis=0)) assert_eq(np.take(x, [3, 4, 5], axis=-1), da.take(a, [3, 4, 5], axis=-1)) with pytest.raises(ValueError): da.take(a, 3, axis=2) assert same_keys(da.take(a, [3, 4, 5], axis=-1), da.take(a, [3, 4, 5], axis=-1))
def reduction_1d_test(da_func, darr, np_func, narr, use_dtype=True, split_every=True): assert_eq(da_func(darr), np_func(narr)) assert_eq( da_func(narr), np_func(narr) ) # Ensure Dask reductions work with NumPy arrays assert_eq(da_func(darr, keepdims=True), np_func(narr, keepdims=True)) assert_eq(da_func(darr, axis=()), np_func(narr, axis=())) assert same_keys(da_func(darr), da_func(darr)) assert same_keys(da_func(darr, keepdims=True), da_func(darr, keepdims=True)) if use_dtype: assert_eq(da_func(darr, dtype="f8"), np_func(narr, dtype="f8")) assert_eq(da_func(darr, dtype="i8"), np_func(narr, dtype="i8")) assert same_keys(da_func(darr, dtype="i8"), da_func(darr, dtype="i8")) if split_every: a1 = da_func(darr, split_every=2) a2 = da_func(darr, split_every={0: 2}) assert same_keys(a1, a2) assert_eq(a1, np_func(narr)) assert_eq(a2, np_func(narr)) assert_eq( da_func(darr, keepdims=True, split_every=2), np_func(narr, keepdims=True) )
def test_fromfunction(func, dtype, kwargs): a = np.fromfunction(func, shape=(5, 5), dtype=dtype, **kwargs) d = da.fromfunction( func, shape=(5, 5), chunks=(2, 2), dtype=dtype, **kwargs ) assert_eq(d, a) d2 = da.fromfunction( func, shape=(5, 5), chunks=(2, 2), dtype=dtype, **kwargs ) assert same_keys(d, d2)
def test_histogram(): # Test for normal, flattened input n = 100 v = da.random.random(n, chunks=10) bins = np.arange(0, 1.01, 0.01) (a1, b1) = da.histogram(v, bins=bins) (a2, b2) = np.histogram(v, bins=bins) # Check if the sum of the bins equals the number of samples assert a2.sum(axis=0) == n assert a1.sum(axis=0) == n assert_eq(a1, a2) assert same_keys(da.histogram(v, bins=bins)[0], a1)
def test_ghost(): x = np.arange(64).reshape((8, 8)) d = da.from_array(x, chunks=(4, 4)) g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'reflect'}) assert g.chunks == ((8, 8), (6, 6)) expected = np.array( [[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [0, 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 7], [8, 8, 9, 10, 11, 12, 11, 12, 13, 14, 15, 15], [16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23], [24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31], [32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39], [40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47], [16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23], [24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31], [32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39], [40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47], [48, 48, 49, 50, 51, 52, 51, 52, 53, 54, 55, 55], [56, 56, 57, 58, 59, 60, 59, 60, 61, 62, 63, 63], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]]) assert_eq(g, expected) assert same_keys( g, ghost(d, depth={ 0: 2, 1: 1 }, boundary={ 0: 100, 1: 'reflect' })) g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'none'}) expected = np.array([[100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [0, 1, 2, 3, 4, 3, 4, 5, 6, 7], [8, 9, 10, 11, 12, 11, 12, 13, 14, 15], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [48, 49, 50, 51, 52, 51, 52, 53, 54, 55], [56, 57, 58, 59, 60, 59, 60, 61, 62, 63], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100]]) assert_eq(g, expected) assert g.chunks == ((8, 8), (5, 5))
def test_percentile(method): d = da.ones((16, ), chunks=(4, )) qs = [0, 50, 100] assert_eq(da.percentile(d, qs, method=method), np.array([1, 1, 1], dtype=d.dtype)) x = np.array([0, 0, 5, 5, 5, 5, 20, 20]) d = da.from_array(x, chunks=(3, )) result = da.percentile(d, qs, method=method) assert_eq(result, np.array([0, 5, 20], dtype=result.dtype)) assert same_keys(da.percentile(d, qs, method=method), da.percentile(d, qs, method=method)) assert not same_keys(da.percentile(d, qs, method=method), da.percentile(d, [0, 50], method=method)) if method != 'tdigest': x = np.array(['a', 'a', 'd', 'd', 'd', 'e']) d = da.from_array(x, chunks=(3, )) assert_eq(da.percentile(d, [0, 50, 100]), np.array(['a', 'd', 'e'], dtype=x.dtype))
def test_percentile(method): d = da.ones((16,), chunks=(4,)) qs = [0, 50, 100] assert_eq(da.percentile(d, qs, method=method), np.array([1, 1, 1], dtype=d.dtype)) x = np.array([0, 0, 5, 5, 5, 5, 20, 20]) d = da.from_array(x, chunks=(3,)) result = da.percentile(d, qs, method=method) assert_eq(result, np.array([0, 5, 20], dtype=result.dtype)) assert same_keys(da.percentile(d, qs, method=method), da.percentile(d, qs, method=method)) assert not same_keys(da.percentile(d, qs, method=method), da.percentile(d, [0, 50], method=method)) if method != 'tdigest': x = np.array(['a', 'a', 'd', 'd', 'd', 'e']) d = da.from_array(x, chunks=(3,)) assert_eq(da.percentile(d, [0, 50, 100]), np.array(['a', 'd', 'e'], dtype=x.dtype))
def test_percentile_with_categoricals(): try: import pandas as pd except ImportError: return x0 = pd.Categorical(["Alice", "Bob", "Charlie", "Dennis", "Alice", "Alice"]) x1 = pd.Categorical(["Alice", "Bob", "Charlie", "Dennis", "Alice", "Alice"]) dsk = {("x", 0): x0, ("x", 1): x1} x = da.Array(dsk, "x", chunks=((6, 6),)) p = da.percentile(x, [50]) assert (p.compute().categories == x0.categories).all() assert (p.compute().codes == [0]).all() assert same_keys(da.percentile(x, [50]), da.percentile(x, [50]))
def test_overlap(): x = np.arange(64).reshape((8, 8)) d = da.from_array(x, chunks=(4, 4)) g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'reflect'}) assert g.chunks == ((8, 8), (6, 6)) expected = np.array( [[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [ 0, 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 7], [ 8, 8, 9, 10, 11, 12, 11, 12, 13, 14, 15, 15], [ 16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23], [ 24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31], [ 32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39], [ 40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47], [ 16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23], [ 24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31], [ 32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39], [ 40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47], [ 48, 48, 49, 50, 51, 52, 51, 52, 53, 54, 55, 55], [ 56, 56, 57, 58, 59, 60, 59, 60, 61, 62, 63, 63], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]]) assert_eq(g, expected) assert same_keys(g, overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'reflect'})) g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'none'}) expected = np.array( [[100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [ 0, 1, 2, 3, 4, 3, 4, 5, 6, 7], [ 8, 9, 10, 11, 12, 11, 12, 13, 14, 15], [ 16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [ 24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [ 32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [ 40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [ 16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [ 24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [ 32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [ 40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [ 48, 49, 50, 51, 52, 51, 52, 53, 54, 55], [ 56, 57, 58, 59, 60, 59, 60, 61, 62, 63], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100]]) assert_eq(g, expected) assert g.chunks == ((8, 8), (5, 5))
def test_percentile_with_categoricals(): try: import pandas as pd except ImportError: return x0 = pd.Categorical(['Alice', 'Bob', 'Charlie', 'Dennis', 'Alice', 'Alice']) x1 = pd.Categorical(['Alice', 'Bob', 'Charlie', 'Dennis', 'Alice', 'Alice']) dsk = {('x', 0): x0, ('x', 1): x1} x = da.Array(dsk, 'x', chunks=((6, 6),)) p = da.percentile(x, [50]) assert (p.compute().categories == x0.categories).all() assert (p.compute().codes == [0]).all() assert same_keys(da.percentile(x, [50]), da.percentile(x, [50]))
def test_percentile_with_categoricals(): try: import pandas as pd except ImportError: return x0 = pd.Categorical( ['Alice', 'Bob', 'Charlie', 'Dennis', 'Alice', 'Alice']) x1 = pd.Categorical( ['Alice', 'Bob', 'Charlie', 'Dennis', 'Alice', 'Alice']) dsk = {('x', 0): x0, ('x', 1): x1} x = da.Array(dsk, 'x', chunks=((6, 6), )) p = da.percentile(x, [50]) assert (p.compute().categories == x0.categories).all() assert (p.compute().codes == [0]).all() assert same_keys(da.percentile(x, [50]), da.percentile(x, [50]))
def test_overlap_internal_asymmetric_small(): x = np.arange(32).reshape((2, 16)) d = da.from_array(x, chunks=(2, 4)) result = overlap_internal(d, {0: (0, 0), 1: (1, 1)}) assert result.chunks == ((2, ), (5, 6, 6, 5)) expected = np.array([[ 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 8, 7, 8, 9, 10, 11, 12, 11, 12, 13, 14, 15 ], [ 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 24, 23, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31 ]]) assert_eq(result, expected) assert same_keys(overlap_internal(d, {0: (0, 0), 1: (1, 1)}), result)
def test_overlap_internal_asymmetric(): x = np.arange(64).reshape((8, 8)) d = da.from_array(x, chunks=(4, 4)) result = overlap_internal(d, {0: (2, 0), 1: (1, 0)}) assert result.chunks == ((6, 4), (5, 4)) expected = np.array([[0, 1, 2, 3, 3, 4, 5, 6, 7], [8, 9, 10, 11, 11, 12, 13, 14, 15], [16, 17, 18, 19, 19, 20, 21, 22, 23], [24, 25, 26, 27, 27, 28, 29, 30, 31], [16, 17, 18, 19, 19, 20, 21, 22, 23], [24, 25, 26, 27, 27, 28, 29, 30, 31], [32, 33, 34, 35, 35, 36, 37, 38, 39], [40, 41, 42, 43, 43, 44, 45, 46, 47], [48, 49, 50, 51, 51, 52, 53, 54, 55], [56, 57, 58, 59, 59, 60, 61, 62, 63]]) assert_eq(result, expected) assert same_keys(overlap_internal(d, {0: (2, 0), 1: (1, 0)}), result)
def test_percentile(): d = da.from_array(cupy.ones((16,)), chunks=(4,)) qs = np.array([0, 50, 100]) assert_eq( da.percentile(d, qs, interpolation="midpoint"), np.array([1, 1, 1], dtype=d.dtype), ) x = cupy.array([0, 0, 5, 5, 5, 5, 20, 20]) d = da.from_array(x, chunks=(3,)) result = da.percentile(d, qs, interpolation="midpoint") assert_eq(result, np.array([0, 5, 20], dtype=result.dtype)) assert not same_keys( da.percentile(d, qs, interpolation="midpoint"), da.percentile(d, [0, 50], interpolation="midpoint"), )
def test_squeeze(is_func, axis): a = np.arange(10)[None, :, None, None] d = da.from_array(a, chunks=(1, 3, 1, 1)) if is_func: a_s = np.squeeze(a, axis=axis) d_s = da.squeeze(d, axis=axis) else: a_s = a.squeeze(axis=axis) d_s = d.squeeze(axis=axis) assert_eq(d_s, a_s) assert same_keys(d_s, da.squeeze(d, axis=axis)) if axis is None: axis = tuple(range(a.ndim)) else: axis = axis if isinstance(axis, tuple) else (axis,) axis = tuple(i % a.ndim for i in axis) axis = tuple(i for i, c in enumerate(d.chunks) if i in axis and len(c) == 1) exp_d_s_chunks = tuple(c for i, c in enumerate(d.chunks) if i not in axis) assert d_s.chunks == exp_d_s_chunks
def test_ghost_internal(): x = np.arange(64).reshape((8, 8)) d = da.from_array(x, chunks=(4, 4)) g = ghost_internal(d, {0: 2, 1: 1}) result = g.compute(get=get) assert g.chunks == ((6, 6), (5, 5)) expected = np.array([[0, 1, 2, 3, 4, 3, 4, 5, 6, 7], [8, 9, 10, 11, 12, 11, 12, 13, 14, 15], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [48, 49, 50, 51, 52, 51, 52, 53, 54, 55], [56, 57, 58, 59, 60, 59, 60, 61, 62, 63]]) assert_eq(result, expected) assert same_keys(ghost_internal(d, {0: 2, 1: 1}), g)
def test_insert(): x = np.random.randint(10, size=(10, 10)) a = da.from_array(x, chunks=(5, 5)) y = np.random.randint(10, size=(5, 10)) b = da.from_array(y, chunks=(4, 4)) assert_eq(np.insert(x, 0, -1, axis=0), da.insert(a, 0, -1, axis=0)) assert_eq(np.insert(x, 3, -1, axis=-1), da.insert(a, 3, -1, axis=-1)) assert_eq(np.insert(x, 5, -1, axis=1), da.insert(a, 5, -1, axis=1)) assert_eq(np.insert(x, -1, -1, axis=-2), da.insert(a, -1, -1, axis=-2)) assert_eq(np.insert(x, [2, 3, 3], -1, axis=1), da.insert(a, [2, 3, 3], -1, axis=1)) assert_eq( np.insert(x, [2, 3, 8, 8, -2, -2], -1, axis=0), da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0), ) assert_eq(np.insert(x, slice(1, 4), -1, axis=1), da.insert(a, slice(1, 4), -1, axis=1)) assert_eq( np.insert(x, [2] * 3 + [5] * 2, y, axis=0), da.insert(a, [2] * 3 + [5] * 2, b, axis=0), ) assert_eq(np.insert(x, 0, y[0], axis=1), da.insert(a, 0, b[0], axis=1)) assert same_keys( da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0), da.insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0), ) with pytest.raises(NotImplementedError): da.insert(a, [4, 2], -1, axis=0) with pytest.raises(AxisError): da.insert(a, [3], -1, axis=2) with pytest.raises(AxisError): da.insert(a, [3], -1, axis=-3)
def test_overlap_internal(): x = cupy.arange(64).reshape((8, 8)) d = da.from_array(x, chunks=(4, 4), asarray=False) g = da.overlap.overlap_internal(d, {0: 2, 1: 1}) assert g.chunks == ((6, 6), (5, 5)) expected = np.array([ [0, 1, 2, 3, 4, 3, 4, 5, 6, 7], [8, 9, 10, 11, 12, 11, 12, 13, 14, 15], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [48, 49, 50, 51, 52, 51, 52, 53, 54, 55], [56, 57, 58, 59, 60, 59, 60, 61, 62, 63], ]) assert_eq(g, expected) assert same_keys(da.overlap.overlap_internal(d, {0: 2, 1: 1}), g)
def test_slicing_consistent_names(): x = np.arange(100).reshape((10, 10)) a = da.from_array(x, chunks=(5, 5)) assert same_keys(a[0], a[0]) assert same_keys(a[:, [1, 2, 3]], a[:, [1, 2, 3]]) assert same_keys(a[:, 5:2:-1], a[:, 5:2:-1])
def test_overlap(): x = np.arange(64).reshape((8, 8)) d = da.from_array(x, chunks=(4, 4)) g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: "reflect"}) assert g.chunks == ((8, 8), (6, 6)) expected = np.array([ [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [0, 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 7], [8, 8, 9, 10, 11, 12, 11, 12, 13, 14, 15, 15], [16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23], [24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31], [32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39], [40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47], [16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23], [24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31], [32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39], [40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47], [48, 48, 49, 50, 51, 52, 51, 52, 53, 54, 55, 55], [56, 56, 57, 58, 59, 60, 59, 60, 61, 62, 63, 63], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], ]) assert_eq(g, expected) assert same_keys( g, overlap(d, depth={ 0: 2, 1: 1 }, boundary={ 0: 100, 1: "reflect" })) u_depth = np.uint16([2, 1]) u_depth = {k: v for k, v in enumerate(u_depth)} g = overlap(d, depth=u_depth, boundary={0: 100, 1: "reflect"}) assert g.chunks == ((8, 8), (6, 6)) assert_eq(g, expected) assert same_keys( g, overlap(d, depth={ 0: 2, 1: 1 }, boundary={ 0: 100, 1: "reflect" })) g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: "none"}) expected = np.array([ [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [0, 1, 2, 3, 4, 3, 4, 5, 6, 7], [8, 9, 10, 11, 12, 11, 12, 13, 14, 15], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [16, 17, 18, 19, 20, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 43, 44, 45, 46, 47], [48, 49, 50, 51, 52, 51, 52, 53, 54, 55], [56, 57, 58, 59, 60, 59, 60, 61, 62, 63], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], ]) assert_eq(g, expected) assert g.chunks == ((8, 8), (5, 5)) u_depth = np.uint16([2, 1]) u_depth = {k: v for k, v in enumerate(u_depth)} g = overlap(d, depth=u_depth, boundary={0: 100, 1: "none"}) assert_eq(g, expected) assert g.chunks == ((8, 8), (5, 5))
def test_slicing_consistent_names_after_normalization(): x = da.zeros(10, chunks=(5,)) assert same_keys(x[0:], x[:10]) assert same_keys(x[0:], x[0:10]) assert same_keys(x[0:], x[0:10:1]) assert same_keys(x[:], x[0:10:1])
def test_fft_consistent_names(funcname): da_fft = getattr(da.fft, funcname) assert same_keys(da_fft(darr, 5), da_fft(darr, 5)) assert same_keys(da_fft(darr2, 5, axis=0), da_fft(darr2, 5, axis=0)) assert not same_keys(da_fft(darr, 5), da_fft(darr, 13))
def test_percentile_tokenize(): d = da.from_array(cupy.ones((16,)), chunks=(4,)) qs = np.array([0, 50, 100]) assert same_keys(da.percentile(d, qs), da.percentile(d, qs))
def test_bincount(): x = np.array([2, 1, 5, 2, 1]) d = da.from_array(x, chunks=2) e = da.bincount(d, minlength=6) assert_eq(e, np.bincount(x, minlength=6)) assert same_keys(da.bincount(d, minlength=6), e)