def test_atop_legacy():
x = da.ones(10, chunks=(5,))
with pytest.warns(None):
y = da.atop(inc, 'i', x, 'i', dtype=x.dtype)
z = da.blockwise(inc, 'i', x, 'i', dtype=x.dtype)
assert_eq(y, z)
assert y.name == z.name
def test_arange_working_float_step():
"""Sometimes floating point step arguments work, but this could be platform
dependent.
"""
darr = da.arange(3.3, -9.1, -.25, chunks=3)
nparr = np.arange(3.3, -9.1, -.25)
assert_eq(darr, nparr)
def test_blockwise_no_array_args():
def f(dtype):
return np.ones((2, 3), dtype)
x = da.blockwise(f, 'ab', np.float32, None, new_axes={'a': 2, 'b': (3, 3)}, dtype=np.float32)
assert x.chunks == ((2,), (3, 3))
assert_eq(x, np.ones((2, 6), np.float32))
def test_array_ufunc():
x = np.arange(24).reshape((4, 6))
d = da.from_array(x, chunks=(2, 3))
for func in [np.sin, np.isreal, np.sum, np.negative, partial(np.prod, axis=0)]:
assert isinstance(func(d), da.Array)
assert_eq(func(d), func(x))
def test_squeeze():
x = da.ones((10, 1), chunks=(3, 1))
assert_eq(x.squeeze(), x.compute().squeeze())
assert x.squeeze().chunks == ((3, 3, 3, 1),)
assert same_keys(x.squeeze(), x.squeeze())
def test_moments(k):
x = np.random.random(size=(30, 2))
y = da.from_array(x, 3)
expected = scipy.stats.moment(x, k)
result = dask.array.stats.moment(y, k)
assert_eq(result, expected)
def test_histogram_return_type():
v = da.random.random(100, chunks=10)
bins = np.arange(0, 1.01, 0.01)
# Check if return type is same as hist
bins = np.arange(0, 11, 1, dtype='i4')
assert_eq(da.histogram(v * 10, bins=bins)[0],
np.histogram(v * 10, bins=bins)[0])
def test_isclose():
x = np.array([0, np.nan, 1, 1.5])
y = np.array([1e-9, np.nan, 1, 2])
a = da.from_array(x, chunks=(2,))
b = da.from_array(y, chunks=(2,))
assert_eq(da.isclose(a, b, equal_nan=True),
np.isclose(x, y, equal_nan=True))
def test_squeeze(is_func, axis):
a = np.arange(10)[None, :, None, None]
d = da.from_array(a, chunks=(1, 3, 1, 1))
if is_func:
a_s = np.squeeze(a, axis=axis)
d_s = da.squeeze(d, axis=axis)
else:
a_s = a.squeeze(axis=axis)
d_s = d.squeeze(axis=axis)
assert_eq(d_s, a_s)
assert same_keys(d_s, da.squeeze(d, axis=axis))
if axis is None:
axis = tuple(range(a.ndim))
else:
axis = axis if isinstance(axis, tuple) else (axis,)
axis = tuple(i % a.ndim for i in axis)
axis = tuple(
i for i, c in enumerate(d.chunks) if i in axis and len(c) == 1
)
exp_d_s_chunks = tuple(
c for i, c in enumerate(d.chunks) if i not in axis
)
assert d_s.chunks == exp_d_s_chunks
def test_unique_kwargs(return_index, return_inverse, return_counts):
kwargs = dict(
return_index=return_index,
return_inverse=return_inverse,
return_counts=return_counts
)
a = np.array([1, 2, 4, 4, 5, 2])
d = da.from_array(a, chunks=(3,))
r_a = np.unique(a, **kwargs)
r_d = da.unique(d, **kwargs)
if not any([return_index, return_inverse, return_counts]):
assert isinstance(r_a, np.ndarray)
assert isinstance(r_d, da.Array)
r_a = (r_a,)
r_d = (r_d,)
assert len(r_a) == len(r_d)
if return_inverse:
i = 1 + int(return_index)
assert (d.size,) == r_d[i].shape
for e_r_a, e_r_d in zip(r_a, r_d):
assert_eq(e_r_d, e_r_a)
def test_einsum_order(order):
sig = 'ea,fb,abcd,gc,hd->efgh'
input_sigs = sig.split('->')[0].split(',')
np_inputs, da_inputs = _numpy_and_dask_inputs(input_sigs)
assert_eq(np.einsum(sig, *np_inputs, order=order),
da.einsum(sig, *np_inputs, order=order))
def test_dot_method():
x = np.arange(400).reshape((20, 20))
a = da.from_array(x, chunks=(5, 5))
y = np.arange(200).reshape((20, 10))
b = da.from_array(y, chunks=(5, 5))
assert_eq(a.dot(b), x.dot(y))
def test_flip(funcname, kwargs, shape):
if (funcname == "flip" and
LooseVersion(np.__version__) < LooseVersion("1.12.0")):
pytest.skip(
"NumPy %s doesn't support `flip`."
" Need NumPy 1.12.0 or greater." % np.__version__
)
axis = kwargs.get("axis")
if axis is None:
if funcname == "flipud":
axis = 0
elif funcname == "fliplr":
axis = 1
np_a = np.random.random(shape)
da_a = da.from_array(np_a, chunks=1)
np_func = getattr(np, funcname)
da_func = getattr(da, funcname)
try:
range(np_a.ndim)[axis]
except IndexError:
with pytest.raises(ValueError):
da_func(da_a, **kwargs)
else:
np_r = np_func(np_a, **kwargs)
da_r = da_func(da_a, **kwargs)
assert_eq(np_r, da_r)
def test_einsum_casting(casting):
sig = 'ea,fb,abcd,gc,hd->efgh'
input_sigs = sig.split('->')[0].split(',')
np_inputs, da_inputs = _numpy_and_dask_inputs(input_sigs)
assert_eq(np.einsum(sig, *np_inputs, casting=casting),
da.einsum(sig, *np_inputs, casting=casting))
def test_ravel_1D_no_op():
x = np.random.randint(10, size=100)
dx = da.from_array(x, chunks=10)
# known dims
assert_eq(dx.ravel(), x.ravel())
# Unknown dims
assert_eq(dx[dx > 2].ravel(), x[x > 2].ravel())
def test_histogram_bins_range_with_nan_array():
# Regression test for issue #3977
v = da.from_array(np.array([-2, np.nan, 2]), chunks=1)
(a1, b1) = da.histogram(v, bins=10, range=(-3, 3))
(a2, b2) = np.histogram(v, bins=10, range=(-3, 3))
assert_eq(a1, a2)
assert_eq(b1, b2)
def test_average(a, returned):
d_a = da.from_array(a, chunks=2)
np_avg = np.average(a, returned=returned)
da_avg = da.average(d_a, returned=returned)
assert_eq(np_avg, da_avg)
def test_norm_1dim(shape, chunks, axis, norm, keepdims):
a = np.random.random(shape)
d = da.from_array(a, chunks=chunks)
a_r = np.linalg.norm(a, ord=norm, axis=axis, keepdims=keepdims)
d_r = da.linalg.norm(d, ord=norm, axis=axis, keepdims=keepdims)
assert_eq(a_r, d_r)
def test_atop_kwargs():
def f(a, b=0):
return a + b
x = da.ones(5, chunks=(2,))
y = atop(f, 'i', x, 'i', b=10, dtype=x.dtype)
assert_eq(y, np.ones(5) + 10)
def test_serializability():
state = da.random.RandomState(5)
x = state.normal(10, 1, size=10, chunks=5)
y = _loads(_dumps(x))
assert_eq(x, y)
def test_overlap_internal():
x = np.arange(64).reshape((8, 8))
d = da.from_array(x, chunks=(4, 4))
g = overlap_internal(d, {0: 2, 1: 1})
result = g.compute(scheduler='sync')
assert g.chunks == ((6, 6), (5, 5))
expected = np.array([
[ 0, 1, 2, 3, 4, 3, 4, 5, 6, 7],
[ 8, 9, 10, 11, 12, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 51, 52, 53, 54, 55],
[56, 57, 58, 59, 60, 59, 60, 61, 62, 63]])
assert_eq(result, expected)
assert same_keys(overlap_internal(d, {0: 2, 1: 1}), g)
def test_bag_array_conversion():
import dask.bag as db
b = db.range(10, npartitions=1)
x, = b.map_partitions(np.asarray).to_delayed()
x, = [da.from_delayed(a, shape=(10,), dtype=int) for a in [x]]
z = da.concatenate([x])
assert_eq(z, np.arange(10), check_graph=False)
def test_apply_along_axis(func1d_name, func1d, shape, axis):
a = np.random.randint(0, 10, shape)
d = da.from_array(a, chunks=(len(shape) * (5,)))
assert_eq(
da.apply_along_axis(func1d, axis, d),
np.apply_along_axis(func1d, axis, a)
)
def test_linspace(endpoint):
darr = da.linspace(6, 49, endpoint=endpoint, chunks=5)
nparr = np.linspace(6, 49, endpoint=endpoint)
assert_eq(darr, nparr)
darr = da.linspace(1.4, 4.9, endpoint=endpoint, chunks=5, num=13)
nparr = np.linspace(1.4, 4.9, endpoint=endpoint, num=13)
assert_eq(darr, nparr)
darr = da.linspace(6, 49, endpoint=endpoint, chunks=5, dtype=float)
nparr = np.linspace(6, 49, endpoint=endpoint, dtype=float)
assert_eq(darr, nparr)
darr, dstep = da.linspace(6, 49, endpoint=endpoint, chunks=5, retstep=True)
nparr, npstep = np.linspace(6, 49, endpoint=endpoint, retstep=True)
assert np.allclose(dstep, npstep)
assert_eq(darr, nparr)
darr = da.linspace(1.4, 4.9, endpoint=endpoint, chunks=5, num=13, dtype=int)
nparr = np.linspace(1.4, 4.9, num=13, endpoint=endpoint, dtype=int)
assert_eq(darr, nparr)
assert (sorted(da.linspace(1.4, 4.9, endpoint=endpoint, chunks=5, num=13).dask) ==
sorted(da.linspace(1.4, 4.9, endpoint=endpoint, chunks=5, num=13).dask))
assert (sorted(da.linspace(6, 49, endpoint=endpoint, chunks=5, dtype=float).dask) ==
sorted(da.linspace(6, 49, endpoint=endpoint, chunks=5, dtype=float).dask))
def test_meshgrid(shapes, chunks, indexing, sparse):
xi_a = []
xi_d = []
xi_dc = []
for each_shape, each_chunk in zip(shapes, chunks):
xi_a.append(np.random.random(each_shape))
xi_d_e = da.from_array(xi_a[-1], chunks=each_chunk)
xi_d.append(xi_d_e)
xi_d_ef = xi_d_e.flatten()
xi_dc.append(xi_d_ef.chunks[0])
do = list(range(len(xi_dc)))
if indexing == "xy" and len(xi_dc) > 1:
do[0], do[1] = do[1], do[0]
xi_dc[0], xi_dc[1] = xi_dc[1], xi_dc[0]
xi_dc = tuple(xi_dc)
r_a = np.meshgrid(*xi_a, indexing=indexing, sparse=sparse)
r_d = da.meshgrid(*xi_d, indexing=indexing, sparse=sparse)
assert isinstance(r_d, list)
assert len(r_a) == len(r_d)
for e_r_a, e_r_d, i in zip(r_a, r_d, do):
assert_eq(e_r_a, e_r_d)
if sparse:
assert e_r_d.chunks[i] == xi_dc[i]
else:
assert e_r_d.chunks == xi_dc
def test_gufunc_mixed_inputs():
def foo(x, y):
return x + y
a = np.ones((2, 1), dtype=int)
b = da.ones((1, 8), chunks=(2, 3), dtype=int)
x = apply_gufunc(foo, "(),()->()", a, b, output_dtypes=int)
assert_eq(x, 2 * np.ones((2, 8), dtype=int))
def test_apply_gufunc_elemwise_01():
def add(x, y):
return x + y
a = da.from_array(np.array([1, 2, 3]), chunks=2, name='a')
b = da.from_array(np.array([1, 2, 3]), chunks=2, name='b')
z = apply_gufunc(add, "(),()->()", a, b, output_dtypes=a.dtype)
assert_eq(z, np.array([2, 4, 6]))
def test_gufunc_two_inputs():
def foo(x, y):
return np.einsum('...ij,...jk->ik', x, y)
a = da.ones((2, 3), chunks=100, dtype=int)
b = da.ones((3, 4), chunks=100, dtype=int)
x = apply_gufunc(foo, "(i,j),(j,k)->(i,k)", a, b, output_dtypes=int)
assert_eq(x, 3 * np.ones((2, 4), dtype=int))
def test_bincount_unspecified_minlength():
x = np.array([1, 1, 3, 7, 0])
d = da.from_array(x, chunks=2)
e = da.bincount(d)
assert_eq(e, np.bincount(x))
assert same_keys(da.bincount(d), e)
assert len(e.compute()) == 8 # shape is (nan,) so must compute for len()
def test_fromfunction():
def f(x, y):
return x + y
d = da.fromfunction(f, shape=(5, 5), chunks=(2, 2), dtype='f8')
assert_eq(d, np.fromfunction(f, shape=(5, 5)))
assert same_keys(d, da.fromfunction(f, shape=(5, 5), chunks=(2, 2), dtype='f8'))
def test_empty_slice():
x = da.ones((5, 5), chunks=(2, 2), dtype='i4')
y = x[:0]
assert_eq(y, np.ones((5, 5), dtype='i4')[:0])
def test_no_chunks_svd():
x = np.random.random((100, 10))
u, s, v = np.linalg.svd(x, full_matrices=False)
for chunks in [((np.nan,) * 10, (10,)), ((np.nan,) * 10, (np.nan,))]:
dx = da.from_array(x, chunks=(10, 10))
dx._chunks = chunks
du, ds, dv = da.linalg.svd(dx)
assert_eq(s, ds)
assert_eq(u.dot(np.diag(s)).dot(v), du.dot(da.diag(ds)).dot(dv))
assert_eq(du.T.dot(du), np.eye(10))
assert_eq(dv.T.dot(dv), np.eye(10))
dx = da.from_array(x, chunks=(10, 10))
dx._chunks = ((np.nan,) * 10, (np.nan,))
assert_eq(abs(v), abs(dv))
assert_eq(abs(u), abs(du))
def test_lstsq(nrow, ncol, chunk, iscomplex):
np.random.seed(1)
A = np.random.randint(1, 20, (nrow, ncol))
b = np.random.randint(1, 20, nrow)
if iscomplex:
A = A + 1.0j * np.random.randint(1, 20, A.shape)
b = b + 1.0j * np.random.randint(1, 20, b.shape)
dA = da.from_array(A, (chunk, ncol))
db = da.from_array(b, chunk)
x, r, rank, s = np.linalg.lstsq(A, b, rcond=-1)
dx, dr, drank, ds = da.linalg.lstsq(dA, db)
assert_eq(dx, x)
assert_eq(dr, r)
assert drank.compute() == rank
assert_eq(ds, s)
# reduce rank causes multicollinearity, only compare rank
A[:, 1] = A[:, 2]
dA = da.from_array(A, (chunk, ncol))
db = da.from_array(b, chunk)
x, r, rank, s = np.linalg.lstsq(
A, b, rcond=np.finfo(np.double).eps * max(nrow, ncol)
)
assert rank == ncol - 1
dx, dr, drank, ds = da.linalg.lstsq(dA, db)
assert drank.compute() == rank
# 2D case
A = np.random.randint(1, 20, (nrow, ncol))
b2D = np.random.randint(1, 20, (nrow, ncol // 2))
if iscomplex:
A = A + 1.0j * np.random.randint(1, 20, A.shape)
b2D = b2D + 1.0j * np.random.randint(1, 20, b2D.shape)
dA = da.from_array(A, (chunk, ncol))
db2D = da.from_array(b2D, (chunk, ncol // 2))
x, r, rank, s = np.linalg.lstsq(A, b2D, rcond=-1)
dx, dr, drank, ds = da.linalg.lstsq(dA, db2D)
assert_eq(dx, x)
assert_eq(dr, r)
assert drank.compute() == rank
assert_eq(ds, s)
def test_slice_list_then_None():
x = da.zeros(shape=(5, 5), chunks=(3, 3))
y = x[[2, 1]][None]
assert_eq(y, np.zeros((1, 2, 5)))
def test_multiple_list_slicing():
x = np.random.rand(6, 7, 8)
a = da.from_array(x, chunks=(3, 3, 3))
assert_eq(x[:, [0, 1, 2]][[0, 1]], a[:, [0, 1, 2]][[0, 1]])
def test_tsqr_zero_height_chunks():
m_q = 10
n_q = 5
m_r = 5
n_r = 5
# certainty
mat = np.random.rand(10, 5)
x = da.from_array(mat, chunks=((4, 0, 1, 0, 5), (5,)))
q, r = da.linalg.qr(x)
assert_eq((m_q, n_q), q.shape) # shape check
assert_eq((m_r, n_r), r.shape) # shape check
assert_eq(mat, da.dot(q, r)) # accuracy check
assert_eq(np.eye(n_q, n_q), da.dot(q.T, q)) # q must be orthonormal
assert_eq(r, da.triu(r.rechunk(r.shape[0]))) # r must be upper triangular
# uncertainty
mat2 = np.vstack([mat, -(np.ones((10, 5)))])
v2 = mat2[:, 0]
x2 = da.from_array(mat2, chunks=5)
c = da.from_array(v2, chunks=5)
x = x2[c >= 0, :] # remove the ones added above to yield mat
q, r = da.linalg.qr(x)
q = q.compute() # because uncertainty
r = r.compute()
assert_eq((m_q, n_q), q.shape) # shape check
assert_eq((m_r, n_r), r.shape) # shape check
assert_eq(mat, np.dot(q, r)) # accuracy check
assert_eq(np.eye(n_q, n_q), np.dot(q.T, q)) # q must be orthonormal
assert_eq(r, np.triu(r)) # r must be upper triangular
def test_tsqr_uncertain(m_min, n_max, chunks, vary_rows, vary_cols, error_type):
mat = np.random.rand(m_min * 2, n_max)
m, n = m_min * 2, n_max
mat[0:m_min, 0] += 1
_c0 = mat[:, 0]
_r0 = mat[0, :]
c0 = da.from_array(_c0, chunks=m_min, name="c")
r0 = da.from_array(_r0, chunks=n_max, name="r")
data = da.from_array(mat, chunks=chunks, name="A")
if vary_rows:
data = data[c0 > 0.5, :]
mat = mat[_c0 > 0.5, :]
m = mat.shape[0]
if vary_cols:
data = data[:, r0 > 0.5]
mat = mat[:, _r0 > 0.5]
n = mat.shape[1]
# qr
m_q = m
n_q = min(m, n)
m_r = n_q
n_r = n
# svd
m_u = m
n_u = min(m, n)
n_s = n_q
m_vh = n_q
n_vh = n
d_vh = max(m_vh, n_vh) # full matrix returned
if error_type is None:
# test QR
q, r = tsqr(data)
q = q.compute() # because uncertainty
r = r.compute()
assert_eq((m_q, n_q), q.shape) # shape check
assert_eq((m_r, n_r), r.shape) # shape check
assert_eq(mat, np.dot(q, r)) # accuracy check
assert_eq(np.eye(n_q, n_q), np.dot(q.T, q)) # q must be orthonormal
assert_eq(r, np.triu(r)) # r must be upper triangular
# test SVD
u, s, vh = tsqr(data, compute_svd=True)
u = u.compute() # because uncertainty
s = s.compute()
vh = vh.compute()
s_exact = np.linalg.svd(mat)[1]
assert_eq(s, s_exact) # s must contain the singular values
assert_eq((m_u, n_u), u.shape) # shape check
assert_eq((n_s,), s.shape) # shape check
assert_eq((d_vh, d_vh), vh.shape) # shape check
assert_eq(np.eye(n_u, n_u), np.dot(u.T, u)) # u must be orthonormal
assert_eq(np.eye(d_vh, d_vh), np.dot(vh, vh.T)) # vh must be orthonormal
assert_eq(mat, np.dot(np.dot(u, np.diag(s)), vh[:n_q])) # accuracy check
else:
with pytest.raises(error_type):
q, r = tsqr(data)
with pytest.raises(error_type):
u, s, vh = tsqr(data, compute_svd=True)
def test_negative_list_slicing():
x = np.arange(5)
dx = da.from_array(x, chunks=2)
assert_eq(dx[[0, -5]], x[[0, -5]])
assert_eq(dx[[4, -1]], x[[4, -1]])
def test_negative_n_slicing():
assert_eq(da.ones(2, chunks=2)[-2], np.ones(2)[-2])
def test_slicing_with_Nones(shape, index):
x = np.random.random(shape)
d = da.from_array(x, chunks=shape)
assert_eq(x[index], d[index])
def test_solve(shape, chunk):
np.random.seed(1)
A = np.random.randint(1, 10, (shape, shape))
dA = da.from_array(A, (chunk, chunk))
# vector
b = np.random.randint(1, 10, shape)
db = da.from_array(b, chunk)
res = da.linalg.solve(dA, db)
assert_eq(res, scipy.linalg.solve(A, b))
assert_eq(dA.dot(res), b.astype(float))
# tall-and-skinny matrix
b = np.random.randint(1, 10, (shape, 5))
db = da.from_array(b, (chunk, 5))
res = da.linalg.solve(dA, db)
assert_eq(res, scipy.linalg.solve(A, b))
assert_eq(dA.dot(res), b.astype(float))
# matrix
b = np.random.randint(1, 10, (shape, shape))
db = da.from_array(b, (chunk, chunk))
res = da.linalg.solve(dA, db)
assert_eq(res, scipy.linalg.solve(A, b))
assert_eq(dA.dot(res), b.astype(float))
def test_getter():
result = da.core.getter(cupy.arange(5), (None, slice(None, None)))
assert type(result) == cupy.ndarray
assert_eq(result, np.arange(5)[None, :], check_type=False)
def test_setitem_extended_API_2d_rhs_func_of_lhs():
# Cases:
# * RHS and/or indices are a function of the LHS
# * Indices have unknown chunk sizes
# * RHS has extra leading size 1 dimensions compared to LHS
x = cupy.arange(60).reshape((6, 10))
chunks = (2, 3)
dx = da.from_array(x, chunks=chunks)
dx[2:4, dx[0] > 3] = -5
x[2:4, x[0] > 3] = -5
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[2, dx[0] < -2] = -7
x[2, x[0] < -2] = -7
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[dx % 2 == 0] = -8
x[x % 2 == 0] = -8
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[dx % 2 == 0] = -8
x[x % 2 == 0] = -8
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[3:5, 5:1:-2] = -dx[:2, 4:1:-2]
x[3:5, 5:1:-2] = -x[:2, 4:1:-2]
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[0, 1:3] = -dx[0, 4:2:-1]
x[0, 1:3] = -x[0, 4:2:-1]
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[...] = dx
x[...] = x
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[...] = dx[...]
x[...] = x[...]
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[0] = dx[-1]
x[0] = x[-1]
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[0, :] = dx[-2, :]
x[0, :] = x[-2, :]
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[:, 1] = dx[:, -3]
x[:, 1] = x[:, -3]
assert_eq(x, dx.compute())
index = da.from_array([0, 2], chunks=(2, ))
dx = da.from_array(x, chunks=chunks)
dx[index, 8] = [99, 88]
x[[0, 2], 8] = [99, 88]
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=chunks)
dx[:, index] = dx[:, :2]
x[:, [0, 2]] = x[:, :2]
assert_eq(x, dx.compute())
index = da.where(da.arange(3, chunks=(1, )) < 2)[0]
dx = da.from_array(x, chunks=chunks)
dx[index, 7] = [-23, -33]
x[index.compute(), 7] = [-23, -33]
assert_eq(x, dx.compute())
index = da.where(da.arange(3, chunks=(1, )) < 2)[0]
dx = da.from_array(x, chunks=chunks)
dx[(index, )] = -34
x[(index.compute(), )] = -34
assert_eq(x, dx.compute())
index = index - 4
dx = da.from_array(x, chunks=chunks)
dx[index, 7] = [-43, -53]
x[index.compute(), 7] = [-43, -53]
assert_eq(x, dx.compute())
index = da.from_array([0, -1], chunks=(1, ))
x[[0, -1]] = 9999
dx[(index, )] = 9999
assert_eq(x, dx.compute())
dx = da.from_array(x, chunks=(-1, -1))
dx[...] = da.from_array(x, chunks=chunks)
assert_eq(x, dx.compute())
# Both tests below fail in CuPy due to leading singular dimensions
if False:
# RHS has extra leading size 1 dimensions compared to LHS
dx = da.from_array(x.copy(), chunks=(2, 3))
v = x.reshape((1, 1) + x.shape)
x[...] = v
dx[...] = v
assert_eq(x, dx.compute())
index = da.where(da.arange(3, chunks=(1, )) < 2)[0]
v = -cupy.arange(12).reshape(1, 1, 6, 2)
x[:, [0, 1]] = v
dx[:, index] = v
assert_eq(x, dx.compute())
def test_slice_stop_0():
# from gh-125
a = da.ones(10, chunks=(10,))[:0].compute()
b = np.ones(10)[:0]
assert_eq(a, b)
def test_arange():
darr = da.arange(77, chunks=13)
nparr = np.arange(77)
assert_eq(darr, nparr)
darr = da.arange(2, 13, chunks=5)
nparr = np.arange(2, 13)
assert_eq(darr, nparr)
darr = da.arange(4, 21, 9, chunks=13)
nparr = np.arange(4, 21, 9)
assert_eq(darr, nparr)
# negative steps
darr = da.arange(53, 5, -3, chunks=5)
nparr = np.arange(53, 5, -3)
assert_eq(darr, nparr)
darr = da.arange(77, chunks=13, dtype=float)
nparr = np.arange(77, dtype=float)
assert_eq(darr, nparr)
darr = da.arange(2, 13, chunks=5, dtype=int)
nparr = np.arange(2, 13, dtype=int)
assert_eq(darr, nparr)
assert (sorted(da.arange(2, 13, chunks=5).dask) == sorted(
da.arange(2, 13, chunks=5).dask))
assert (sorted(da.arange(77, chunks=13, dtype=float).dask) == sorted(
da.arange(77, chunks=13, dtype=float).dask))
# 0 size output
darr = da.arange(0, 1, -0.5, chunks=20)
nparr = np.arange(0, 1, -0.5)
assert_eq(darr, nparr)
darr = da.arange(0, -1, 0.5, chunks=20)
nparr = np.arange(0, -1, 0.5)
assert_eq(darr, nparr)
def test_diag():
v = np.arange(11)
assert_eq(da.diag(v), np.diag(v))
v = da.arange(11, chunks=3)
darr = da.diag(v)
nparr = np.diag(v)
assert_eq(darr, nparr)
assert sorted(da.diag(v).dask) == sorted(da.diag(v).dask)
v = v + v + 3
darr = da.diag(v)
nparr = np.diag(v)
assert_eq(darr, nparr)
v = da.arange(11, chunks=11)
darr = da.diag(v)
nparr = np.diag(v)
assert_eq(darr, nparr)
assert sorted(da.diag(v).dask) == sorted(da.diag(v).dask)
x = np.arange(64).reshape((8, 8))
assert_eq(da.diag(x), np.diag(x))
d = da.from_array(x, chunks=(4, 4))
assert_eq(da.diag(d), np.diag(x))
def test_tile(shape, chunks, reps):
x = np.random.random(shape)
d = da.from_array(x, chunks=chunks)
assert_eq(np.tile(x, reps), da.tile(d, reps))
def test_view():
x = np.arange(56).reshape((7, 8))
d = da.from_array(cupy.array(x), chunks=(2, 3))
result = d.view()
assert type(result._meta) == cupy.ndarray
assert_eq(result,
result) # Check that _meta and computed arrays match types
assert_eq(result, x.view(), check_type=False)
result = d.view("i4")
assert type(result._meta) == cupy.ndarray
assert_eq(result,
result) # Check that _meta and computed arrays match types
assert_eq(result, x.view("i4"), check_type=False)
result = d.view("i2")
assert type(result._meta) == cupy.ndarray
assert_eq(result,
result) # Check that _meta and computed arrays match types
assert_eq(result, x.view("i2"), check_type=False)
assert all(isinstance(s, int) for s in d.shape)
x = np.arange(8, dtype="i1")
d = da.from_array(cupy.array(x), chunks=(4, ))
result = d.view("i4")
assert type(result._meta) == cupy.ndarray
assert_eq(result,
result) # Check that _meta and computed arrays match types
assert_eq(x.view("i4"), d.view("i4"), check_type=False)
with pytest.raises(ValueError):
x = np.arange(8, dtype="i1")
d = da.from_array(cupy.array(x), chunks=(3, ))
d.view("i4")
with pytest.raises(ValueError):
d.view("i4", order="asdf")
def test_tril_triu_non_square_arrays():
A = np.random.randint(0, 11, (30, 35))
dA = da.from_array(A, chunks=(5, 5))
assert_eq(da.triu(dA), np.triu(A))
assert_eq(da.tril(dA), np.tril(A))
def test_determinisim_through_dask_values():
samples_1 = da.random.RandomState(42).normal(size=1000, chunks=10)
samples_2 = da.random.RandomState(42).normal(size=1000, chunks=10)
assert set(samples_1.dask) == set(samples_2.dask)
assert_eq(samples_1, samples_2)
def test_arange_cast_float_int_step():
darr = da.arange(3.3, -9.1, -.25, chunks=3, dtype='i8')
nparr = np.arange(3.3, -9.1, -.25, dtype='i8')
assert_eq(darr, nparr)
def test_eye():
assert_eq(da.eye(9, chunks=3), np.eye(9))
assert_eq(da.eye(10, chunks=3), np.eye(10))
assert_eq(da.eye(9, chunks=3, M=11), np.eye(9, M=11))
assert_eq(da.eye(11, chunks=3, M=9), np.eye(11, M=9))
assert_eq(da.eye(7, chunks=3, M=11), np.eye(7, M=11))
assert_eq(da.eye(11, chunks=3, M=7), np.eye(11, M=7))
assert_eq(da.eye(9, chunks=3, k=2), np.eye(9, k=2))
assert_eq(da.eye(9, chunks=3, k=-2), np.eye(9, k=-2))
assert_eq(da.eye(7, chunks=3, M=11, k=5), np.eye(7, M=11, k=5))
assert_eq(da.eye(11, chunks=3, M=7, k=-6), np.eye(11, M=7, k=-6))
assert_eq(da.eye(6, chunks=3, M=9, k=7), np.eye(6, M=9, k=7))
assert_eq(da.eye(12, chunks=3, M=6, k=-3), np.eye(12, M=6, k=-3))
assert_eq(da.eye(9, chunks=3, dtype=int), np.eye(9, dtype=int))
assert_eq(da.eye(10, chunks=3, dtype=int), np.eye(10, dtype=int))
def test_map_overlap_multiarray_different_depths():
x = da.ones(5, dtype="int")
y = da.ones(5, dtype="int")
def run(depth):
return da.map_overlap(
lambda x, y: x.sum() + y.sum(), x, y, depth=depth, chunks=(0,), trim=False
).compute()
# Check that the number of elements added
# to arrays in overlap works as expected
# when depths differ for each array
assert_eq(run([0, 0]), 10)
assert_eq(run([0, 1]), 12)
assert_eq(run([1, 1]), 14)
assert_eq(run([1, 2]), 16)
assert_eq(run([0, 5]), 20)
assert_eq(run([5, 5]), 30)
# Ensure that depth > chunk size results in error
with pytest.raises(ValueError):
run([0, 6])
def test_map_overlap():
x = da.arange(10, chunks=5)
y = x.map_overlap(lambda x: x + len(x), depth=2, dtype=x.dtype)
assert_eq(y, np.arange(10) + 5 + 2 + 2)
x = da.arange(10, chunks=5)
y = x.map_overlap(lambda x: x + len(x), depth=np.int64(2), dtype=x.dtype)
assert all([(type(s) is int) for s in y.shape])
assert_eq(y, np.arange(10) + 5 + 2 + 2)
x = np.arange(16).reshape((4, 4))
d = da.from_array(x, chunks=(2, 2))
exp1 = d.map_overlap(lambda x: x + x.size, depth=1, dtype=d.dtype)
exp2 = d.map_overlap(
lambda x: x + x.size,
depth={0: 1, 1: 1},
boundary={0: "reflect", 1: "none"},
dtype=d.dtype,
)
exp3 = d.map_overlap(
lambda x: x + x.size, depth={1: 1}, boundary={1: "reflect"}, dtype=d.dtype
)
exp4 = d.map_overlap(
lambda x: x + x.size,
depth={1: (1, 0)},
boundary={0: "none", 1: "none"},
dtype=d.dtype,
)
assert_eq(exp1, x + 16)
assert_eq(exp2, x + 12)
assert_eq(exp3, x + 8)
assert_eq(
exp4,
np.block(
[[x[0:2, 0:2] + 4, x[0:2, 2:4] + 6], [x[2:4, 0:2] + 4, x[2:4, 2:4] + 6]]
),
)
def test_map_overlap_no_depth(boundary):
x = da.arange(10, chunks=5)
y = x.map_overlap(lambda i: i, depth=0, boundary=boundary, dtype=x.dtype)
assert_eq(y, x)
def test_consistent_across_sizes():
x1 = da.random.RandomState(123).random(20, chunks=20)
x2 = da.random.RandomState(123).random(100, chunks=20)[:20]
x3 = da.random.RandomState(123).random(200, chunks=20)[:20]
assert_eq(x1, x2)
assert_eq(x1, x3)
def test_tsqr(m, n, chunks, error_type):
mat = np.random.rand(m, n)
data = da.from_array(mat, chunks=chunks, name="A")
# qr
m_q = m
n_q = min(m, n)
m_r = n_q
n_r = n
# svd
m_u = m
n_u = min(m, n)
n_s = n_q
m_vh = n_q
n_vh = n
d_vh = max(m_vh, n_vh) # full matrix returned
if error_type is None:
# test QR
q, r = tsqr(data)
assert_eq((m_q, n_q), q.shape) # shape check
assert_eq((m_r, n_r), r.shape) # shape check
assert_eq(mat, da.dot(q, r)) # accuracy check
assert_eq(np.eye(n_q, n_q), da.dot(q.T, q)) # q must be orthonormal
assert_eq(r, da.triu(r.rechunk(r.shape[0]))) # r must be upper triangular
# test SVD
u, s, vh = tsqr(data, compute_svd=True)
s_exact = np.linalg.svd(mat)[1]
assert_eq(s, s_exact) # s must contain the singular values
assert_eq((m_u, n_u), u.shape) # shape check
assert_eq((n_s,), s.shape) # shape check
assert_eq((d_vh, d_vh), vh.shape) # shape check
assert_eq(np.eye(n_u, n_u), da.dot(u.T, u)) # u must be orthonormal
assert_eq(np.eye(d_vh, d_vh), da.dot(vh, vh.T)) # vh must be orthonormal
assert_eq(mat, da.dot(da.dot(u, da.diag(s)), vh[:n_q])) # accuracy check
else:
with pytest.raises(error_type):
q, r = tsqr(data)
with pytest.raises(error_type):
u, s, vh = tsqr(data, compute_svd=True)
def test_solve_sym_pos(shape, chunk):
np.random.seed(1)
A = _get_symmat(shape)
dA = da.from_array(A, (chunk, chunk))
# vector
b = np.random.randint(1, 10, shape)
db = da.from_array(b, chunk)
res = da.linalg.solve(dA, db, sym_pos=True)
assert_eq(res, scipy.linalg.solve(A, b, sym_pos=True), check_graph=False)
assert_eq(dA.dot(res), b.astype(float), check_graph=False)
# tall-and-skinny matrix
b = np.random.randint(1, 10, (shape, 5))
db = da.from_array(b, (chunk, 5))
res = da.linalg.solve(dA, db, sym_pos=True)
assert_eq(res, scipy.linalg.solve(A, b, sym_pos=True), check_graph=False)
assert_eq(dA.dot(res), b.astype(float), check_graph=False)
# matrix
b = np.random.randint(1, 10, (shape, shape))
db = da.from_array(b, (chunk, chunk))
res = da.linalg.solve(dA, db, sym_pos=True)
assert_eq(res, scipy.linalg.solve(A, b, sym_pos=True), check_graph=False)
assert_eq(dA.dot(res), b.astype(float), check_graph=False)
def _check_lu_result(p, l, u, A):
assert np.allclose(p.dot(l).dot(u), A)
# check triangulars
assert_eq(l, da.tril(l), check_graph=False)
assert_eq(u, da.triu(u), check_graph=False)
def test_overlap():
x = np.arange(64).reshape((8, 8))
d = da.from_array(x, chunks=(4, 4))
g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: "reflect"})
assert g.chunks == ((8, 8), (6, 6))
expected = np.array(
[
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[0, 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 7],
[8, 8, 9, 10, 11, 12, 11, 12, 13, 14, 15, 15],
[16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23],
[24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31],
[32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39],
[40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47],
[16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23],
[24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31],
[32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39],
[40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47],
[48, 48, 49, 50, 51, 52, 51, 52, 53, 54, 55, 55],
[56, 56, 57, 58, 59, 60, 59, 60, 61, 62, 63, 63],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
]
)
assert_eq(g, expected)
assert same_keys(g, overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: "reflect"}))
u_depth = np.uint16([2, 1])
u_depth = {k: v for k, v in enumerate(u_depth)}
g = overlap(d, depth=u_depth, boundary={0: 100, 1: "reflect"})
assert g.chunks == ((8, 8), (6, 6))
assert_eq(g, expected)
assert same_keys(g, overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: "reflect"}))
g = overlap(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: "none"})
expected = np.array(
[
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[0, 1, 2, 3, 4, 3, 4, 5, 6, 7],
[8, 9, 10, 11, 12, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[16, 17, 18, 19, 20, 19, 20, 21, 22, 23],
[24, 25, 26, 27, 28, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 51, 52, 53, 54, 55],
[56, 57, 58, 59, 60, 59, 60, 61, 62, 63],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
]
)
assert_eq(g, expected)
assert g.chunks == ((8, 8), (5, 5))
u_depth = np.uint16([2, 1])
u_depth = {k: v for k, v in enumerate(u_depth)}
g = overlap(d, depth=u_depth, boundary={0: 100, 1: "none"})
assert_eq(g, expected)
assert g.chunks == ((8, 8), (5, 5))
